Implement retry that automatically detect httpx and backoff (#119)

* server: implement retry that automatically detect httpx and backoff

Closes #118

* server: fix formatting
This commit is contained in:
2023-08-08 14:03:36 +02:00
committed by GitHub
parent 04d6a2cfae
commit 7d40305737
6 changed files with 139 additions and 45 deletions

View File

@@ -20,6 +20,7 @@ class BananaLLM(LLM):
headers=self.headers,
json={"prompt": prompt},
timeout=self.timeout,
retry_timeout=300, # as per their sdk
)
response.raise_for_status()
text = response.json()["text"]

View File

@@ -21,7 +21,6 @@ class OpenAILLM(LLM):
"Authorization": f"Bearer {self.openai_key}",
}
async with httpx.AsyncClient(timeout=self.timeout) as client:
response = await client.post(
self.openai_url,