server: fixes pipeline logger not transmitted to processors

Closes #110
This commit is contained in:
Mathieu Virbel
2023-08-04 12:02:18 +02:00
parent 6d2085ce61
commit dce92e0cf7
6 changed files with 32 additions and 9 deletions

View File

@@ -17,7 +17,8 @@ class Processor:
self.logger = (custom_logger or logger).bind(processor=self.__class__.__name__)
def set_pipeline(self, pipeline: "Pipeline"):
self.logger = self.logger.bind(pipeline=pipeline.uid)
# if pipeline is used, pipeline logger will be used instead
self.logger = pipeline.logger.bind(processor=self.__class__.__name__)
def connect(self, processor: "Processor"):
"""
@@ -111,6 +112,10 @@ class ThreadedProcessor(Processor):
self.queue = asyncio.Queue()
self.task = asyncio.get_running_loop().create_task(self.loop())
def set_pipeline(self, pipeline: "Pipeline"):
super().set_pipeline(pipeline)
self.processor.set_pipeline(pipeline)
async def loop(self):
while True:
data = await self.queue.get()
@@ -153,6 +158,9 @@ class Pipeline(Processor):
def __init__(self, *processors: Processor):
super().__init__()
self.logger = logger.bind(pipeline=self.uid)
self.logger.info("Pipeline created")
self.processors = processors
for processor in processors:
@@ -168,8 +176,10 @@ class Pipeline(Processor):
await self.processors[0].push(data)
async def _flush(self):
self.logger.debug("Pipeline flushing")
for processor in self.processors:
await processor.flush()
self.logger.info("Pipeline flushed")
def describe(self, level=0):
logger.info(" " * level + "Pipeline:")

View File

@@ -1,5 +1,6 @@
from reflector.processors.base import Processor
from reflector.processors.types import Transcript, TitleSummary
from reflector.utils.retry import retry
from reflector.llm import LLM
@@ -42,8 +43,10 @@ class TranscriptTopicDetectorProcessor(Processor):
async def _flush(self):
if not self.transcript:
return
prompt = self.PROMPT.format(input_text=self.transcript.text)
result = await self.llm.generate(prompt=prompt)
text = self.transcript.text
self.logger.info(f"Detect topic on {len(text)} length transcript")
prompt = self.PROMPT.format(input_text=text)
result = await retry(self.llm.generate)(prompt=prompt, logger=self.logger)
summary = TitleSummary(
title=result["title"],
summary=result["summary"],

View File

@@ -67,6 +67,13 @@ class TitleSummary:
duration: float
transcript: Transcript
@property
def human_timestamp(self):
minutes = int(self.timestamp / 60)
seconds = int(self.timestamp % 60)
milliseconds = int((self.timestamp % 1) * 1000)
return f"{minutes:02d}:{seconds:02d}.{milliseconds:03d}"
@dataclass
class FinalSummary: