zen: log error

infra/console.ts

@@ -166,14 +166,10 @@ const bucketNew = new sst.cloudflare.Bucket("ZenDataNew")
 const AWS_SES_ACCESS_KEY_ID = new sst.Secret("AWS_SES_ACCESS_KEY_ID")
 const AWS_SES_SECRET_ACCESS_KEY = new sst.Secret("AWS_SES_SECRET_ACCESS_KEY")
 
-let logProcessor
-if ($app.stage === "production" || $app.stage === "frank") {
-  const HONEYCOMB_API_KEY = new sst.Secret("HONEYCOMB_API_KEY")
-  logProcessor = new sst.cloudflare.Worker("LogProcessor", {
-    handler: "packages/console/function/src/log-processor.ts",
-    link: [HONEYCOMB_API_KEY],
-  })
-}
+const logProcessor = new sst.cloudflare.Worker("LogProcessor", {
+  handler: "packages/console/function/src/log-processor.ts",
+  link: [new sst.Secret("HONEYCOMB_API_KEY")],
+})
 
 new sst.cloudflare.x.SolidStart("Console", {
   domain,
@@ -211,7 +207,7 @@ new sst.cloudflare.x.SolidStart("Console", {
       transform: {
         worker: {
           placement: { mode: "smart" },
-          tailConsumers: logProcessor ? [{ service: logProcessor.nodes.worker.scriptName }] : [],
+          tailConsumers: [{ service: logProcessor.nodes.worker.scriptName }],
         },
       },
     },

packages/console/app/src/routes/zen/util/handler.ts

@@ -134,20 +134,26 @@ export async function handler(
         body: reqBody,
       })
 
-      // Try another provider => stop retrying if using fallback provider
-      if (
-        res.status !== 200 &&
-        // ie. openai 404 error: Item with id 'msg_0ead8b004a3b165d0069436a6b6834819896da85b63b196a3f' not found.
-        res.status !== 404 &&
-        // ie. cannot change codex model providers mid-session
-        modelInfo.stickyProvider !== "strict" &&
-        modelInfo.fallbackProvider &&
-        providerInfo.id !== modelInfo.fallbackProvider
-      ) {
-        return retriableRequest({
-          excludeProviders: [...retry.excludeProviders, providerInfo.id],
-          retryCount: retry.retryCount + 1,
+      if (res.status !== 200) {
+        logger.metric({
+          "llm.error.code": res.status,
+          "llm.error.message": res.statusText,
         })
+
+        // Try another provider => stop retrying if using fallback provider
+        if (
+          // ie. openai 404 error: Item with id 'msg_0ead8b004a3b165d0069436a6b6834819896da85b63b196a3f' not found.
+          res.status !== 404 &&
+          // ie. cannot change codex model providers mid-session
+          modelInfo.stickyProvider !== "strict" &&
+          modelInfo.fallbackProvider &&
+          providerInfo.id !== modelInfo.fallbackProvider
+        ) {
+          return retriableRequest({
+            excludeProviders: [...retry.excludeProviders, providerInfo.id],
+            retryCount: retry.retryCount + 1,
+          })
+        }
       }
 
       return { providerInfo, reqBody, res, startTimestamp }

packages/console/function/src/log-processor.ts

@@ -17,7 +17,7 @@ export default {
       )
         return
 
-      let metrics = {
+      let data = {
         event_type: "completions",
         "cf.continent": event.event.request.cf?.continent,
         "cf.country": event.event.request.cf?.country,
@@ -31,22 +31,28 @@ export default {
         status: event.event.response?.status ?? 0,
         ip: event.event.request.headers["x-real-ip"],
       }
+      const time = event.eventTimestamp ?? Date.now()
+      const events = []
       for (const log of event.logs) {
         for (const message of log.message) {
           if (!message.startsWith("_metric:")) continue
-          metrics = { ...metrics, ...JSON.parse(message.slice(8)) }
+          const json = JSON.parse(message.slice(8))
+          data = { ...data, ...json }
+          if ("llm.error.code" in json) {
+            events.push({ time, data: { ...data } })
+          }
         }
       }
-      console.log(JSON.stringify(metrics, null, 2))
+      events.push({ time, data })
+      console.log(JSON.stringify(data, null, 2))
 
-      const ret = await fetch("https://api.honeycomb.io/1/events/zen", {
+      const ret = await fetch("https://api.honeycomb.io/1/batch/zen", {
         method: "POST",
         headers: {
           "Content-Type": "application/json",
-          "X-Honeycomb-Event-Time": (event.eventTimestamp ?? Date.now()).toString(),
           "X-Honeycomb-Team": Resource.HONEYCOMB_API_KEY.value,
         },
-        body: JSON.stringify(metrics),
+        body: JSON.stringify(events),
       })
       console.log(ret.status)
       console.log(await ret.text())