jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Model: Set default max_batch_size

Browse source
This commit is contained in:
turboderp 2025-05-03 18:32:30 +02:00
parent 8c75b29923
commit 0d949d00b9
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
backends/exllamav3/model.py
View file

@ -180,7 +180,7 @@ class ExllamaV3Container(BaseModelContainer):
self.cache = Cache(self.model, max_num_tokens=self.cache_size)
# Max batch size
self.max_batch_size = kwargs.get("max_batch_size")
self.max_batch_size = unwrap(kwargs.get("max_batch_size"), 256)
# Make sure chunk size is >= 256, keep near or below max seq len
user_chunk_size = unwrap(kwargs.get("chunk_size"), 2048)