Dependencies: Pin tokenizers

Use a version greater than 0.20.0 for newer model support.

Signed-off-by: kingbri <8082010+bdashore3@users.noreply.github.com>
This commit is contained in:
kingbri 2024-12-13 00:58:25 -05:00
parent 1ba33bf646
commit bc3c154c96

View file

@ -25,7 +25,7 @@ dependencies = [
"loguru",
"sse-starlette",
"packaging",
"tokenizers",
"tokenizers>=0.21.0",
"formatron",
"kbnf>=0.4.1",
"aiofiles",