Merge pull request #346 from gakada/main

Exl3: some models aren't functional without add_bos?
2025-05-17 22:05:15 -04:00 · 2025-05-17 22:05:15 -04:00 · 729caaeddc
commit 729caaeddc
parent 0646d358a2 ba6248eec0
1 changed files with 1 additions and 1 deletions
--- a/backends/exllamav3/model.py
+++ b/backends/exllamav3/model.py
@ -844,7 +844,7 @@ class ExllamaV3Container(BaseModelContainer):
        job = AsyncJob(
            self.generator,
            sampler=sampler,
-            input_ids=self.tokenizer.encode(prompt, add_bos=False),
+            input_ids=input_ids,
            max_new_tokens=max_tokens,
            stop_conditions=stop_conditions,
            banned_strings=params.banned_strings,