Model: Cast autosplit_reserve to int
Torch errors if float values are passed (because bytes are not float types). Therefore, overestimate and cast to an int type. Resolves #97 Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
parent
cab789e685
commit
88b0b6f4f1
1 changed files with 5 additions and 1 deletions
|
|
@ -1,6 +1,7 @@
|
|||
"""The model container class for ExLlamaV2 models."""
|
||||
|
||||
import gc
|
||||
import math
|
||||
import pathlib
|
||||
import threading
|
||||
import time
|
||||
|
|
@ -130,7 +131,10 @@ class ExllamaV2Container:
|
|||
|
||||
autosplit_reserve_megabytes = unwrap(kwargs.get("autosplit_reserve"), [96])
|
||||
self.autosplit_reserve = list(
|
||||
map(lambda value: value * 1024**2, autosplit_reserve_megabytes)
|
||||
map(
|
||||
lambda value: int(math.ceil(value * 1024**2)),
|
||||
autosplit_reserve_megabytes,
|
||||
)
|
||||
)
|
||||
elif gpu_count > 1:
|
||||
# Manual GPU split
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue