From 998abe5ad13f29b71230c01aa526d2d1405bc185 Mon Sep 17 00:00:00 2001 From: DocShotgun <126566557+DocShotgun@users.noreply.github.com> Date: Mon, 18 Aug 2025 12:32:28 -0700 Subject: [PATCH 1/4] Config: Enable safe sampler overrides by default * Provides safe fallback samplers, intended for better out-of-the-box support for clients that do not pass sampler params --- .gitignore | 1 + common/config_models.py | 7 +++++-- config_sample.yml | 6 ++++-- sampler_overrides/safe_defaults.yml | 10 ++++++++++ 4 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 sampler_overrides/safe_defaults.yml diff --git a/.gitignore b/.gitignore index 1d71e65..f191c90 100644 --- a/.gitignore +++ b/.gitignore @@ -201,6 +201,7 @@ templates/tool_calls/* # Sampler overrides folder sampler_overrides/* !sampler_overrides/sample_preset.yml +!sampler_overrides/safe_defaults.yml # Gpu lib preferences file gpu_lib.txt diff --git a/common/config_models.py b/common/config_models.py index 9898c04..1017368 100644 --- a/common/config_models.py +++ b/common/config_models.py @@ -406,10 +406,13 @@ class SamplingConfig(BaseConfigModel): override_preset: Optional[str] = Field( None, description=( - "Select a sampler override preset (default: None).\n" + "Select a sampler override preset (default: safe_defaults).\n" "Find this in the sampler-overrides folder.\n" "This overrides default fallbacks for sampler values " - "that are passed to the API." + "that are passed to the API.\n" + "NOTE: safe_defaults preset provides a fallback for frontends " + "that do not pass sampling params.\n" + "Remove it if not necessary." ), ) diff --git a/config_sample.yml b/config_sample.yml index 890596e..652bcd4 100644 --- a/config_sample.yml +++ b/config_sample.yml @@ -208,10 +208,12 @@ embeddings: # Options for Sampling sampling: - # Select a sampler override preset (default: None). + # Select a sampler override preset (default: safe_defaults). # Find this in the sampler-overrides folder. # This overrides default fallbacks for sampler values that are passed to the API. - override_preset: + # NOTE: safe_defaults preset provides a fallback for frontends that do not pass sampling params. + # Remove it if not necessary. + override_preset: safe_defaults # Options for development and experimentation developer: diff --git a/sampler_overrides/safe_defaults.yml b/sampler_overrides/safe_defaults.yml new file mode 100644 index 0000000..d610429 --- /dev/null +++ b/sampler_overrides/safe_defaults.yml @@ -0,0 +1,10 @@ +# Minimal "safe" fallback settings for most models, used when the frontend doesn't supply any settings +# with a request. + +temperature: + override: 0.8 + force: false + +top_p: + override: 0.9 + force: false \ No newline at end of file From 6fb0c2cdbde83c7fc989d997e52efd6e25cd2d60 Mon Sep 17 00:00:00 2001 From: DocShotgun <126566557+DocShotgun@users.noreply.github.com> Date: Mon, 18 Aug 2025 12:39:52 -0700 Subject: [PATCH 2/4] Config: Update description for override_preset default * We provide safe_defaults as a default in config_sample.yml but not internally --- common/config_models.py | 2 +- config_sample.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/config_models.py b/common/config_models.py index 1017368..ad22732 100644 --- a/common/config_models.py +++ b/common/config_models.py @@ -406,7 +406,7 @@ class SamplingConfig(BaseConfigModel): override_preset: Optional[str] = Field( None, description=( - "Select a sampler override preset (default: safe_defaults).\n" + "Select a sampler override preset (default: None).\n" "Find this in the sampler-overrides folder.\n" "This overrides default fallbacks for sampler values " "that are passed to the API.\n" diff --git a/config_sample.yml b/config_sample.yml index 652bcd4..03bb2ab 100644 --- a/config_sample.yml +++ b/config_sample.yml @@ -208,7 +208,7 @@ embeddings: # Options for Sampling sampling: - # Select a sampler override preset (default: safe_defaults). + # Select a sampler override preset (default: None). # Find this in the sampler-overrides folder. # This overrides default fallbacks for sampler values that are passed to the API. # NOTE: safe_defaults preset provides a fallback for frontends that do not pass sampling params. From 067d63773e12f9fac3989c13c8f5c2e8100a2f82 Mon Sep 17 00:00:00 2001 From: kingbri <8082010+kingbri1@users.noreply.github.com> Date: Mon, 18 Aug 2025 22:55:03 -0400 Subject: [PATCH 3/4] Config: Move sampling higher in the list This has become a bigger priority with addition of the safe_defaults noob proofing. Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com> --- common/config_models.py | 34 +++++++++++++++++----------------- config_sample.yml | 18 +++++++++--------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/common/config_models.py b/common/config_models.py index ad22732..bd15592 100644 --- a/common/config_models.py +++ b/common/config_models.py @@ -347,6 +347,23 @@ class DraftModelConfig(BaseConfigModel): ) +class SamplingConfig(BaseConfigModel): + """Options for Sampling""" + + override_preset: Optional[str] = Field( + None, + description=( + "Select a sampler override preset (default: None).\n" + "Find this in the sampler-overrides folder.\n" + "This overrides default fallbacks for sampler values " + "that are passed to the API.\n" + "NOTE: safe_defaults preset provides a fallback for frontends " + "that do not pass sampling params.\n" + "Remove it if not necessary." + ), + ) + + class LoraInstanceModel(BaseConfigModel): """Model representing an instance of a Lora.""" @@ -400,23 +417,6 @@ class EmbeddingsConfig(BaseConfigModel): ) -class SamplingConfig(BaseConfigModel): - """Options for Sampling""" - - override_preset: Optional[str] = Field( - None, - description=( - "Select a sampler override preset (default: None).\n" - "Find this in the sampler-overrides folder.\n" - "This overrides default fallbacks for sampler values " - "that are passed to the API.\n" - "NOTE: safe_defaults preset provides a fallback for frontends " - "that do not pass sampling params.\n" - "Remove it if not necessary." - ), - ) - - class DeveloperConfig(BaseConfigModel): """Options for development and experimentation""" diff --git a/config_sample.yml b/config_sample.yml index 03bb2ab..ad49224 100644 --- a/config_sample.yml +++ b/config_sample.yml @@ -179,6 +179,15 @@ draft_model: # If this isn't filled in, the draft model is autosplit. draft_gpu_split: [] +# Options for Sampling +sampling: + # Select a sampler override preset (default: None). + # Find this in the sampler-overrides folder. + # This overrides default fallbacks for sampler values that are passed to the API. + # NOTE: safe_defaults is noob friendly and provides fallbacks for frontends that don't send sampling parameters. + # Remove this for any advanced usage. + override_preset: safe_defaults + # Options for Loras lora: # Directory to look for LoRAs (default: loras). @@ -206,15 +215,6 @@ embeddings: # An initial embedding model to load on the infinity backend. embedding_model_name: -# Options for Sampling -sampling: - # Select a sampler override preset (default: None). - # Find this in the sampler-overrides folder. - # This overrides default fallbacks for sampler values that are passed to the API. - # NOTE: safe_defaults preset provides a fallback for frontends that do not pass sampling params. - # Remove it if not necessary. - override_preset: safe_defaults - # Options for development and experimentation developer: # Skip Exllamav2 version check (default: False). From e07df3951e73c80ebaa8081964fe87a2484579e2 Mon Sep 17 00:00:00 2001 From: kingbri <8082010+kingbri1@users.noreply.github.com> Date: Mon, 18 Aug 2025 23:06:16 -0400 Subject: [PATCH 4/4] Docs: Update sampler overrides Change the sampling subsection to sampler overrides and add a warning about the default preset. Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com> --- docs/01.-Getting-Started.md | 3 +++ docs/{08.-Sampling.md => 08.-Sampler-Overrides.md} | 0 2 files changed, 3 insertions(+) rename docs/{08.-Sampling.md => 08.-Sampler-Overrides.md} (100%) diff --git a/docs/01.-Getting-Started.md b/docs/01.-Getting-Started.md index c71413a..4bf0b3e 100644 --- a/docs/01.-Getting-Started.md +++ b/docs/01.-Getting-Started.md @@ -76,6 +76,9 @@ A config.yml file is required for overriding project defaults. **If you are okay If you do want a config file, copy over `config_sample.yml` to `config.yml`. All the fields are commented, so make sure to read the descriptions and comment out or remove fields that you don't need. +> [!WARNING] +> Due to frontends not sending sampler settings per request, tabbyAPI sets a safe defaults sampler override in config_sample.yml. If you are testing metrics or experimenting, please remove `safe_defaults` from the `override_preset` key! + In addition, if you want to manually set the API keys, copy over `api_keys_sample.yml` to `api_keys.yml` and fill in the fields. However, doing this is less secure and autogenerated keys should be used instead. You can also access the configuration parameters under [2. Configuration](https://github.com/theroyallab/tabbyAPI/wiki/2.-Configuration) in this wiki! diff --git a/docs/08.-Sampling.md b/docs/08.-Sampler-Overrides.md similarity index 100% rename from docs/08.-Sampling.md rename to docs/08.-Sampler-Overrides.md