jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Config: Clarify Rope alpha options

Browse source 
Leaving blank will use the model's set value or auto-calculate.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri 2024-09-17 23:03:28 -04:00
parent 754fb15f23
commit 63634beb5e
2 changed files with 12 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
common/config_models.py
View file

@ -213,10 +213,12 @@ class ModelConfig(BaseConfigModel):
),
)
rope_alpha: Optional[Union[float, Literal["auto"]]] = Field(
1.0,
None,
description=(
"Rope alpha (default: 1.0).\n"
'Same as alpha_value. Set to "auto" to auto-calculate.'
"Rope alpha (default: None).\n"
'Same as alpha_value. Set to "auto" to auto-calculate.\n'
"Leaving this value blank will either pull from the model "
"or auto-calculate."
),
)
cache_mode: Optional[CACHE_SIZES] = Field(
@ -318,7 +320,9 @@ class DraftModelConfig(BaseConfigModel):
None,
description=(
"Rope alpha for draft models (default: None).\n"
'Same as alpha_value. Set to "auto" to auto-calculate.'
'Same as alpha_value. Set to "auto" to auto-calculate.\n'
"Leaving this value blank will either pull from the model "
"or auto-calculate."
),
)
draft_cache_mode: Optional[CACHE_SIZES] = Field(

6
config_sample.yml
View file

@ -97,9 +97,10 @@ model:
# Leave blank to pull the value from the model.
rope_scale: 1.0
# Rope alpha (default: 1.0).
# Rope alpha (default: None).
# Same as alpha_value. Set to "auto" to auto-calculate.
rope_alpha: 1.0
# Leaving this value blank will either pull from the model or auto-calculate.
rope_alpha:
# Enable different cache modes for VRAM savings (default: FP16).
# Possible values: 'FP16', 'Q8', 'Q6', 'Q4'.
@ -154,6 +155,7 @@ draft_model:
# Rope alpha for draft models (default: None).
# Same as alpha_value. Set to "auto" to auto-calculate.
# Leaving this value blank will either pull from the model or auto-calculate.
draft_rope_alpha:
# Cache mode for draft models to save VRAM (default: FP16).