diff --git a/common/sampling.py b/common/sampling.py index 7e1d810..49be5b9 100644 --- a/common/sampling.py +++ b/common/sampling.py @@ -205,7 +205,7 @@ class BaseSamplerRequest(BaseModel): ) add_bos_token: Optional[bool] = Field( - default_factory=lambda: get_default_sampler_value("add_bos_token", True) + default_factory=lambda: get_default_sampler_value("add_bos_token") ) ban_eos_token: Optional[bool] = Field( diff --git a/endpoints/OAI/types/chat_completion.py b/endpoints/OAI/types/chat_completion.py index fb73eb9..36934a8 100644 --- a/endpoints/OAI/types/chat_completion.py +++ b/endpoints/OAI/types/chat_completion.py @@ -84,7 +84,7 @@ class ChatCompletionRequest(CommonCompletionRequest): # Chat completions requests do not have a BOS token preference. Backend # respects the tokenization config for the individual model. - add_bos_token: Optional[bool] = Field(default=None) + add_bos_token: Optional[bool] = None @field_validator("add_bos_token", mode="after") def force_bos_token(cls, v): diff --git a/sampler_overrides/sample_preset.yml b/sampler_overrides/sample_preset.yml index 907bbdd..0a2d05c 100644 --- a/sampler_overrides/sample_preset.yml +++ b/sampler_overrides/sample_preset.yml @@ -131,7 +131,7 @@ mirostat_eta: # MARK: Token options add_bos_token: - override: true + override: force: false ban_eos_token: override: false