diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py index fdb85a7..044bce0 100644 --- a/backends/exllamav2/model.py +++ b/backends/exllamav2/model.py @@ -1085,11 +1085,13 @@ class ExllamaV2Container: # < 0 = disabled if dry_multiplier > 0: + gen_settings.dry_multiplier = dry_multiplier + + # TODO: Maybe set the "sane" defaults instead? gen_settings.dry_allowed_length = unwrap( kwargs.get("dry_allowed_length"), 0 ) - gen_settings.dry_base = unwrap(kwargs.get("dry_base"), 2.0) - gen_settings.dry_multiplier = unwrap(kwargs.get("dry_multiplier"), 2.0) + gen_settings.dry_base = unwrap(kwargs.get("dry_base"), 0.0) # Exl2 has dry_range as 0 for unlimited unlike -1 for penalty_range # Use max_seq_len as the fallback to stay consistent diff --git a/common/sampling.py b/common/sampling.py index de5b7dc..67b8925 100644 --- a/common/sampling.py +++ b/common/sampling.py @@ -141,16 +141,16 @@ class BaseSamplerRequest(BaseModel): default_factory=lambda: get_default_sampler_value("repetition_decay", 0) ) - dry_allowed_length: Optional[int] = Field( - default_factory=lambda: get_default_sampler_value("dry_allowed_length", 0) + dry_multiplier: Optional[float] = Field( + default_factory=lambda: get_default_sampler_value("dry_multiplier", 0.0) ) dry_base: Optional[float] = Field( - default_factory=lambda: get_default_sampler_value("dry_base", 2.0) + default_factory=lambda: get_default_sampler_value("dry_base", 0.0) ) - dry_multiplier: Optional[float] = Field( - default_factory=lambda: get_default_sampler_value("dry_multiplier", 2.0) + dry_allowed_length: Optional[int] = Field( + default_factory=lambda: get_default_sampler_value("dry_allowed_length", 0) ) dry_range: Optional[int] = Field( @@ -369,9 +369,9 @@ class BaseSamplerRequest(BaseModel): "presence_penalty": self.presence_penalty, "repetition_penalty": self.repetition_penalty, "penalty_range": self.penalty_range, - "dry_allowed_length": self.dry_allowed_length, - "dry_base": self.dry_base, "dry_multiplier": self.dry_multiplier, + "dry_base": self.dry_base, + "dry_allowed_length": self.dry_allowed_length, "dry_sequence_breakers": self.dry_sequence_breakers, "repetition_decay": self.repetition_decay, "mirostat": self.mirostat_mode == 2, diff --git a/sampler_overrides/sample_preset.yml b/sampler_overrides/sample_preset.yml index b20b042..ed2edba 100644 --- a/sampler_overrides/sample_preset.yml +++ b/sampler_overrides/sample_preset.yml @@ -97,6 +97,20 @@ penalty_range: override: -1 force: false +# MARK: DRY +dry_multiplier: + override: 0.0 + force: false +dry_base: + override: 0.0 + force: false +dry_allowed_length: + override: 0 + force: false +dry_range: + override: 0 + force: false + # MARK: Mirostat mirostat_mode: override: 0