selfhostyourtech/reflector
2
0
Fork 0
mirror of https://github.com/Monadical-SAS/reflector.git synced 2025-12-20 20:29:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix loading shards from local cache (#313)

Browse Source
This commit is contained in:
projects-g
2023-11-08 22:02:48 +05:30
committed by GitHub
parent 6282583d92
commit 5cb132cac7
3 changed files with 11 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/gpu/modal/reflector_llm.py
View File

@@ -81,7 +81,8 @@ class LLM:
LLM_MODEL,
torch_dtype=getattr(torch, LLM_TORCH_DTYPE),
low_cpu_mem_usage=LLM_LOW_CPU_MEM_USAGE,
cache_dir=IMAGE_MODEL_DIR
cache_dir=IMAGE_MODEL_DIR,
local_files_only=True
)
# JSONFormer doesn't yet support generation configs
@@ -96,7 +97,8 @@ class LLM:
print("Instance llm tokenizer")
tokenizer = AutoTokenizer.from_pretrained(
LLM_MODEL,
cache_dir=IMAGE_MODEL_DIR
cache_dir=IMAGE_MODEL_DIR,
local_files_only=True
)
# move model to gpu