jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Model: Fix draft model non-FA2 fallback

Browse source
This commit is contained in:
DocShotgun 2024-11-15 21:04:25 -08:00
parent 101ebd658a
commit 5bb46df3c3
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
backends/exllamav2/model.py
View file

@ -159,7 +159,6 @@ class ExllamaV2Container:
if enable_draft:
self.draft_config = ExLlamaV2Config()
self.draft_config.no_flash_attn = self.config.no_flash_attn
draft_model_path = pathlib.Path(
unwrap(draft_args.get("draft_model_dir"), "models")
)
@ -253,6 +252,8 @@ class ExllamaV2Container:
or not supports_paged_attn()
):
self.config.no_flash_attn = True
if self.draft_config:
self.draft_config.no_flash_attn = True
self.paged = False
self.max_batch_size = 1
torch.backends.cuda.enable_flash_sdp(False)