diff --git a/.gitignore b/.gitignore index 1d71e65..f191c90 100644 --- a/.gitignore +++ b/.gitignore @@ -201,6 +201,7 @@ templates/tool_calls/* # Sampler overrides folder sampler_overrides/* !sampler_overrides/sample_preset.yml +!sampler_overrides/safe_defaults.yml # Gpu lib preferences file gpu_lib.txt diff --git a/common/config_models.py b/common/config_models.py index 9898c04..bd15592 100644 --- a/common/config_models.py +++ b/common/config_models.py @@ -347,6 +347,23 @@ class DraftModelConfig(BaseConfigModel): ) +class SamplingConfig(BaseConfigModel): + """Options for Sampling""" + + override_preset: Optional[str] = Field( + None, + description=( + "Select a sampler override preset (default: None).\n" + "Find this in the sampler-overrides folder.\n" + "This overrides default fallbacks for sampler values " + "that are passed to the API.\n" + "NOTE: safe_defaults preset provides a fallback for frontends " + "that do not pass sampling params.\n" + "Remove it if not necessary." + ), + ) + + class LoraInstanceModel(BaseConfigModel): """Model representing an instance of a Lora.""" @@ -400,20 +417,6 @@ class EmbeddingsConfig(BaseConfigModel): ) -class SamplingConfig(BaseConfigModel): - """Options for Sampling""" - - override_preset: Optional[str] = Field( - None, - description=( - "Select a sampler override preset (default: None).\n" - "Find this in the sampler-overrides folder.\n" - "This overrides default fallbacks for sampler values " - "that are passed to the API." - ), - ) - - class DeveloperConfig(BaseConfigModel): """Options for development and experimentation""" diff --git a/config_sample.yml b/config_sample.yml index 890596e..ad49224 100644 --- a/config_sample.yml +++ b/config_sample.yml @@ -179,6 +179,15 @@ draft_model: # If this isn't filled in, the draft model is autosplit. draft_gpu_split: [] +# Options for Sampling +sampling: + # Select a sampler override preset (default: None). + # Find this in the sampler-overrides folder. + # This overrides default fallbacks for sampler values that are passed to the API. + # NOTE: safe_defaults is noob friendly and provides fallbacks for frontends that don't send sampling parameters. + # Remove this for any advanced usage. + override_preset: safe_defaults + # Options for Loras lora: # Directory to look for LoRAs (default: loras). @@ -206,13 +215,6 @@ embeddings: # An initial embedding model to load on the infinity backend. embedding_model_name: -# Options for Sampling -sampling: - # Select a sampler override preset (default: None). - # Find this in the sampler-overrides folder. - # This overrides default fallbacks for sampler values that are passed to the API. - override_preset: - # Options for development and experimentation developer: # Skip Exllamav2 version check (default: False). diff --git a/docs/01.-Getting-Started.md b/docs/01.-Getting-Started.md index c71413a..4bf0b3e 100644 --- a/docs/01.-Getting-Started.md +++ b/docs/01.-Getting-Started.md @@ -76,6 +76,9 @@ A config.yml file is required for overriding project defaults. **If you are okay If you do want a config file, copy over `config_sample.yml` to `config.yml`. All the fields are commented, so make sure to read the descriptions and comment out or remove fields that you don't need. +> [!WARNING] +> Due to frontends not sending sampler settings per request, tabbyAPI sets a safe defaults sampler override in config_sample.yml. If you are testing metrics or experimenting, please remove `safe_defaults` from the `override_preset` key! + In addition, if you want to manually set the API keys, copy over `api_keys_sample.yml` to `api_keys.yml` and fill in the fields. However, doing this is less secure and autogenerated keys should be used instead. You can also access the configuration parameters under [2. Configuration](https://github.com/theroyallab/tabbyAPI/wiki/2.-Configuration) in this wiki! diff --git a/docs/08.-Sampling.md b/docs/08.-Sampler-Overrides.md similarity index 100% rename from docs/08.-Sampling.md rename to docs/08.-Sampler-Overrides.md diff --git a/sampler_overrides/safe_defaults.yml b/sampler_overrides/safe_defaults.yml new file mode 100644 index 0000000..d610429 --- /dev/null +++ b/sampler_overrides/safe_defaults.yml @@ -0,0 +1,10 @@ +# Minimal "safe" fallback settings for most models, used when the frontend doesn't supply any settings +# with a request. + +temperature: + override: 0.8 + force: false + +top_p: + override: 0.9 + force: false \ No newline at end of file