jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Warning if unsupported samplers are used

Browse source
This commit is contained in:
turboderp 2023-11-21 18:35:22 +01:00
parent a54de11cf3
commit 3337fe6acc
1 changed files with 17 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

17
model.py
View file

@ -270,6 +270,23 @@ class ModelContainer:
# Sampler settings
gen_settings = ExLlamaV2Sampler.Settings()
# Warn if unsupported settings supplied
if "mirostat" in kwargs and not hasattr(gen_settings, "mirostat"):
print(" !! Warning: Currently installed ExLlamaV2 does not support Mirostat sampling")
if kwargs.get("min_p", 0.0) not in [0.0, 1.0] and not hasattr(gen_settings, "min_p"):
print(" !! Warning: Currently installed ExLlamaV2 does not support min-P sampling")
if kwargs.get("tfs", 0.0) not in [0.0, 1.0] and not hasattr(gen_settings, "tfs"):
print(" !! Warning: Currently installed ExLlamaV2 does not support tail-free sampling (TFS)")
if "temperature_last" in kwargs and not hasattr(gen_settings, "temperature_last"):
print(" !! Warning: Currently installed ExLlamaV2 does not support temperature_last")
#Apply settings
gen_settings.temperature = kwargs.get("temperature", 1.0)
gen_settings.temperature_last = kwargs.get("temperature_last", False)
gen_settings.top_k = kwargs.get("top_k", 1)