jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #11 from veden/patch-1

Browse source 
Fix incorrect ratio calculation for draft model
This commit is contained in:
turboderp 2023-11-20 04:23:34 +01:00 • committed by GitHub
commit 8ef730f016
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
model.py
View file

@ -101,8 +101,6 @@ class ModelContainer:
self.draft_config.model_dir = str(draft_model_path.resolve())
self.draft_config.prepare()
self.draft_config.max_seq_len = self.config.max_seq_len
if "draft_rope_alpha" in kwargs:
self.draft_config.scale_alpha_value = kwargs.get("draft_rope_alpha") or 1
else:
@ -110,6 +108,8 @@ class ModelContainer:
alpha = -0.13436 + 0.80541 * ratio + 0.28833 * ratio ** 2
self.draft_config.scale_alpha_value = alpha
self.draft_config.max_seq_len = self.config.max_seq_len
if "chunk_size" in kwargs:
self.draft_config.max_input_len = kwargs["chunk_size"]
self.draft_config.max_attn_size = kwargs["chunk_size"] ** 2