Merge pull request #377 from DocShotgun/main

Config: Enable safe sampler overrides by default

.gitignore

@@ -201,6 +201,7 @@ templates/tool_calls/*
 # Sampler overrides folder
 sampler_overrides/*
 !sampler_overrides/sample_preset.yml
+!sampler_overrides/safe_defaults.yml
 
 # Gpu lib preferences file
 gpu_lib.txt

common/config_models.py

@@ -347,6 +347,23 @@ class DraftModelConfig(BaseConfigModel):
     )
 
 
+class SamplingConfig(BaseConfigModel):
+    """Options for Sampling"""
+
+    override_preset: Optional[str] = Field(
+        None,
+        description=(
+            "Select a sampler override preset (default: None).\n"
+            "Find this in the sampler-overrides folder.\n"
+            "This overrides default fallbacks for sampler values "
+            "that are passed to the API.\n"
+            "NOTE: safe_defaults preset provides a fallback for frontends "
+            "that do not pass sampling params.\n"
+            "Remove it if not necessary."
+        ),
+    )
+
+
 class LoraInstanceModel(BaseConfigModel):
     """Model representing an instance of a Lora."""
 
@@ -400,20 +417,6 @@ class EmbeddingsConfig(BaseConfigModel):
     )
 
 
-class SamplingConfig(BaseConfigModel):
-    """Options for Sampling"""
-
-    override_preset: Optional[str] = Field(
-        None,
-        description=(
-            "Select a sampler override preset (default: None).\n"
-            "Find this in the sampler-overrides folder.\n"
-            "This overrides default fallbacks for sampler values "
-            "that are passed to the API."
-        ),
-    )
-
-
 class DeveloperConfig(BaseConfigModel):
     """Options for development and experimentation"""
 

config_sample.yml

@@ -179,6 +179,15 @@ draft_model:
   # If this isn't filled in, the draft model is autosplit.
   draft_gpu_split: []
 
+# Options for Sampling
+sampling:
+  # Select a sampler override preset (default: None).
+  # Find this in the sampler-overrides folder.
+  # This overrides default fallbacks for sampler values that are passed to the API.
+  # NOTE: safe_defaults is noob friendly and provides fallbacks for frontends that don't send sampling parameters.
+  # Remove this for any advanced usage.
+  override_preset: safe_defaults
+
 # Options for Loras
 lora:
   # Directory to look for LoRAs (default: loras).
@@ -206,13 +215,6 @@ embeddings:
   # An initial embedding model to load on the infinity backend.
   embedding_model_name:
 
-# Options for Sampling
-sampling:
-  # Select a sampler override preset (default: None).
-  # Find this in the sampler-overrides folder.
-  # This overrides default fallbacks for sampler values that are passed to the API.
-  override_preset:
-
 # Options for development and experimentation
 developer:
   # Skip Exllamav2 version check (default: False).

docs/01.-Getting-Started.md

@@ -76,6 +76,9 @@ A config.yml file is required for overriding project defaults. **If you are okay
 
 If you do want a config file, copy over `config_sample.yml` to `config.yml`. All the fields are commented, so make sure to read the descriptions and comment out or remove fields that you don't need.
 
+> [!WARNING]
+> Due to frontends not sending sampler settings per request, tabbyAPI sets a safe defaults sampler override in config_sample.yml. If you are testing metrics or experimenting, please remove `safe_defaults` from the `override_preset` key!
+
 In addition, if you want to manually set the API keys, copy over `api_keys_sample.yml` to `api_keys.yml` and fill in the fields. However, doing this is less secure and autogenerated keys should be used instead.
 
 You can also access the configuration parameters under [2. Configuration](https://github.com/theroyallab/tabbyAPI/wiki/2.-Configuration) in this wiki!

sampler_overrides/safe_defaults.yml

@@ -0,0 +1,10 @@
+# Minimal "safe" fallback settings for most models, used when the frontend doesn't supply any settings
+# with a request.
+
+temperature:
+  override: 0.8
+  force: false
+
+top_p:
+  override: 0.9
+  force: false