From f960fac8ff72fd9a785bc97be26cdcddeefa33c0 Mon Sep 17 00:00:00 2001 From: Veden Date: Sun, 19 Nov 2023 13:12:53 -0800 Subject: [PATCH] Fix incorrect ratio calculation for draft model --- model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model.py b/model.py index 2c79f5c..aca6bdc 100644 --- a/model.py +++ b/model.py @@ -101,8 +101,6 @@ class ModelContainer: self.draft_config.model_dir = str(draft_model_path.resolve()) self.draft_config.prepare() - self.draft_config.max_seq_len = self.config.max_seq_len - if "draft_rope_alpha" in kwargs: self.draft_config.scale_alpha_value = kwargs.get("draft_rope_alpha") or 1 else: @@ -110,6 +108,8 @@ class ModelContainer: alpha = -0.13436 + 0.80541 * ratio + 0.28833 * ratio ** 2 self.draft_config.scale_alpha_value = alpha + self.draft_config.max_seq_len = self.config.max_seq_len + if "chunk_size" in kwargs: self.draft_config.max_input_len = kwargs["chunk_size"] self.draft_config.max_attn_size = kwargs["chunk_size"] ** 2