jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Model: Remove old code and fix API handling

Browse source 
skip_special_tokens is in stable exl2. Also default the parameters
if they are not present in the function signature.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri 2024-05-10 21:20:00 -04:00
parent c0b631ba92
commit 7eee936a3f
1 changed files with 2 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

19
backends/exllamav2/model.py
View file

@ -929,24 +929,9 @@ class ExllamaV2Container:
}
)
# Check if decode_special_tokens is supported
# TODO: Remove when a new version of ExllamaV2 is released
if decode_special_tokens:
begin_stream_signature = signature(self.generator.begin_stream_ex)
try:
_bound_vars = begin_stream_signature.bind_partial(
decode_special_tokens=True
)
begin_stream_args["decode_special_tokens"] = decode_special_tokens
except TypeError:
logger.warning(
"skip_special_tokens is not supported by the currently "
"installed ExLlamaV2 version."
)
# MARK: Function signature checks. Not needed in newer ExllamaV2 versions
# Check if temporary token bans are supported
# TODO: Remove when a new version of ExllamaV2 is released
if min_tokens:
stream_signature = signature(self.generator.stream_ex)
@ -962,7 +947,6 @@ class ExllamaV2Container:
min_tokens = 0
# Check if banned_strings is supported
# TODO: Remove when a new version of ExllamaV2 is released
if banned_strings:
begin_stream_signature = signature(self.generator.begin_stream_ex)
@ -976,6 +960,7 @@ class ExllamaV2Container:
"banned_strings is not supported by the currently "
"installed ExLlamaV2 version."
)
banned_strings = []
# Log generation options to console
# Some options are too large, so log the args instead