selfhostyourtech/reflector
2
0
Fork 0
mirror of https://github.com/Monadical-SAS/reflector.git synced 2025-12-21 04:39:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove ast

Browse Source
This commit is contained in:
Gokul Mohanarangan
2023-08-17 15:15:43 +05:30
parent a98a9853be
commit 9103c8cca8
1 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
server/gpu/modal/reflector_llm.py
View File

@@ -3,11 +3,11 @@ Reflector GPU backend - LLM
=========================== ===========================
""" """
import json
import os import os
from typing import Optional from typing import Optional
from modal import asgi_app, Image, method, Secret, Stub from modal import Image, Secret, Stub, asgi_app, method
# LLM # LLM
LLM_MODEL: str = "lmsys/vicuna-13b-v1.5" LLM_MODEL: str = "lmsys/vicuna-13b-v1.5"
@@ -116,12 +116,11 @@ class LLM:
# If a schema is given, conform to schema # If a schema is given, conform to schema
if schema: if schema:
print(f"Schema {schema=}") print(f"Schema {schema=}")
import ast
import jsonformer import jsonformer
jsonformer_llm = jsonformer.Jsonformer(model=self.model, jsonformer_llm = jsonformer.Jsonformer(model=self.model,
tokenizer=self.tokenizer, tokenizer=self.tokenizer,
json_schema=ast.literal_eval(schema), json_schema=json.loads(schema),
prompt=prompt, prompt=prompt,
max_string_token_length=self.gen_cfg.max_new_tokens) max_string_token_length=self.gen_cfg.max_new_tokens)
response = jsonformer_llm() response = jsonformer_llm()
@@ -154,7 +153,7 @@ class LLM:
) )
@asgi_app() @asgi_app()
def web(): def web():
from fastapi import FastAPI, HTTPException, status, Depends from fastapi import Depends, FastAPI, HTTPException, status
from fastapi.security import OAuth2PasswordBearer from fastapi.security import OAuth2PasswordBearer
from pydantic import BaseModel from pydantic import BaseModel