diff --git a/.gitignore b/.gitignore index 1d71e65..f191c90 100644 --- a/.gitignore +++ b/.gitignore @@ -201,6 +201,7 @@ templates/tool_calls/* # Sampler overrides folder sampler_overrides/* !sampler_overrides/sample_preset.yml +!sampler_overrides/safe_defaults.yml # Gpu lib preferences file gpu_lib.txt diff --git a/common/config_models.py b/common/config_models.py index 9898c04..1017368 100644 --- a/common/config_models.py +++ b/common/config_models.py @@ -406,10 +406,13 @@ class SamplingConfig(BaseConfigModel): override_preset: Optional[str] = Field( None, description=( - "Select a sampler override preset (default: None).\n" + "Select a sampler override preset (default: safe_defaults).\n" "Find this in the sampler-overrides folder.\n" "This overrides default fallbacks for sampler values " - "that are passed to the API." + "that are passed to the API.\n" + "NOTE: safe_defaults preset provides a fallback for frontends " + "that do not pass sampling params.\n" + "Remove it if not necessary." ), ) diff --git a/config_sample.yml b/config_sample.yml index 890596e..652bcd4 100644 --- a/config_sample.yml +++ b/config_sample.yml @@ -208,10 +208,12 @@ embeddings: # Options for Sampling sampling: - # Select a sampler override preset (default: None). + # Select a sampler override preset (default: safe_defaults). # Find this in the sampler-overrides folder. # This overrides default fallbacks for sampler values that are passed to the API. - override_preset: + # NOTE: safe_defaults preset provides a fallback for frontends that do not pass sampling params. + # Remove it if not necessary. + override_preset: safe_defaults # Options for development and experimentation developer: diff --git a/sampler_overrides/safe_defaults.yml b/sampler_overrides/safe_defaults.yml new file mode 100644 index 0000000..d610429 --- /dev/null +++ b/sampler_overrides/safe_defaults.yml @@ -0,0 +1,10 @@ +# Minimal "safe" fallback settings for most models, used when the frontend doesn't supply any settings +# with a request. + +temperature: + override: 0.8 + force: false + +top_p: + override: 0.9 + force: false \ No newline at end of file