jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Model: Fix exl2 cache mode check

Browse source 
FP16 was not included in the validation step.

Signed-off-by: kingbri <8082010+kingbri1@users.noreply.github.com>
This commit is contained in:
kingbri 2025-05-12 09:47:49 -04:00
parent 8996dc7b02
commit d15eb55f20
1 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
backends/exllamav2/model.py
View file

@ -188,7 +188,7 @@ class ExllamaV2Container(BaseModelContainer):
self.cache_mode = unwrap(kwargs.get("cache_mode"), "FP16")
# Catch exllamav3 cache_mode
if not self.cache_mode.startswith("Q"):
if not self.cache.mode == "FP16" or not self.cache_mode.startswith("Q"):
logger.warning(
f"Provided cache mode '{self.cache_mode}' is not a "
"valid choice for exllamav2, please check your settings. "
@ -402,7 +402,10 @@ class ExllamaV2Container(BaseModelContainer):
self.draft_cache_mode = unwrap(draft_args.get("draft_cache_mode"), "FP16")
# Catch exllamav3 draft_cache_mode
if not self.draft_cache_mode.startswith("Q"):
if (
not self.draft_cache_mode == "FP16"
or not self.draft_cache_mode.startswith("Q")
):
logger.warning(
f"Provided draft cache mode '{self.draft_cache_mode}' is not a "
"valid choice for exllamav2, please check your settings. "