Merge branch 'main' into mathieu/calendar-integration-rebased

server/pyproject.toml

@@ -26,7 +26,6 @@ dependencies = [
     "prometheus-fastapi-instrumentator>=6.1.0",
     "sentencepiece>=0.1.99",
     "protobuf>=4.24.3",
-    "profanityfilter>=2.0.6",
     "celery>=5.3.4",
     "redis>=5.0.1",
     "python-jose[cryptography]>=3.3.0",

server/reflector/pipelines/main_file_pipeline.py

@@ -12,7 +12,7 @@ from pathlib import Path
 
 import av
 import structlog
-from celery import shared_task
+from celery import chain, shared_task
 
 from reflector.asynctask import asynctask
 from reflector.db.rooms import rooms_controller
@@ -26,6 +26,8 @@ from reflector.logger import logger
 from reflector.pipelines.main_live_pipeline import (
     PipelineMainBase,
     broadcast_to_sockets,
+    task_cleanup_consent,
+    task_pipeline_post_to_zulip,
 )
 from reflector.processors import (
     AudioFileWriterProcessor,
@@ -379,6 +381,28 @@ class PipelineMainFile(PipelineMainBase):
         await processor.flush()
 
 
+@shared_task
+@asynctask
+async def task_send_webhook_if_needed(*, transcript_id: str):
+    """Send webhook if this is a room recording with webhook configured"""
+    transcript = await transcripts_controller.get_by_id(transcript_id)
+    if not transcript:
+        return
+
+    if transcript.source_kind == SourceKind.ROOM and transcript.room_id:
+        room = await rooms_controller.get_by_id(transcript.room_id)
+        if room and room.webhook_url:
+            logger.info(
+                "Dispatching webhook",
+                transcript_id=transcript_id,
+                room_id=room.id,
+                webhook_url=room.webhook_url,
+            )
+            send_transcript_webhook.delay(
+                transcript_id, room.id, event_id=uuid.uuid4().hex
+            )
+
+
 @shared_task
 @asynctask
 async def task_pipeline_file_process(*, transcript_id: str):
@@ -406,16 +430,10 @@ async def task_pipeline_file_process(*, transcript_id: str):
         await pipeline.set_status(transcript_id, "error")
         raise
 
-    # Trigger webhook if this is a room recording with webhook configured
-    if transcript.source_kind == SourceKind.ROOM and transcript.room_id:
-        room = await rooms_controller.get_by_id(transcript.room_id)
-        if room and room.webhook_url:
-            logger.info(
-                "Dispatching webhook task",
-                transcript_id=transcript_id,
-                room_id=room.id,
-                webhook_url=room.webhook_url,
-            )
-            send_transcript_webhook.delay(
-                transcript_id, room.id, event_id=uuid.uuid4().hex
-            )
+    # Run post-processing chain: consent cleanup -> zulip -> webhook
+    post_chain = chain(
+        task_cleanup_consent.si(transcript_id=transcript_id),
+        task_pipeline_post_to_zulip.si(transcript_id=transcript_id),
+        task_send_webhook_if_needed.si(transcript_id=transcript_id),
+    )
+    post_chain.delay()

server/reflector/processors/types.py

@@ -4,11 +4,8 @@ import tempfile
 from pathlib import Path
 from typing import Annotated, TypedDict
 
-from profanityfilter import ProfanityFilter
 from pydantic import BaseModel, Field, PrivateAttr
 
-from reflector.redis_cache import redis_cache
-
 
 class DiarizationSegment(TypedDict):
     """Type definition for diarization segment containing speaker information"""
@@ -20,9 +17,6 @@ class DiarizationSegment(TypedDict):
 
 PUNC_RE = re.compile(r"[.;:?!…]")
 
-profanity_filter = ProfanityFilter()
-profanity_filter.set_censor("*")
-
 
 class AudioFile(BaseModel):
     name: str
@@ -124,21 +118,11 @@ def words_to_segments(words: list[Word]) -> list[TranscriptSegment]:
 
 class Transcript(BaseModel):
     translation: str | None = None
-    words: list[Word] = None
-
-    @property
-    def raw_text(self):
-        # Uncensored text
-        return "".join([word.text for word in self.words])
-
-    @redis_cache(prefix="profanity", duration=3600 * 24 * 7)
-    def _get_censored_text(self, text: str):
-        return profanity_filter.censor(text).strip()
+    words: list[Word] = []
 
     @property
     def text(self):
-        # Censored text
-        return self._get_censored_text(self.raw_text)
+        return "".join([word.text for word in self.words])
 
     @property
     def human_timestamp(self):
@@ -170,12 +154,6 @@ class Transcript(BaseModel):
             word.start += offset
             word.end += offset
 
-    def clone(self):
-        words = [
-            Word(text=word.text, start=word.start, end=word.end) for word in self.words
-        ]
-        return Transcript(text=self.text, translation=self.translation, words=words)
-
     def as_segments(self) -> list[TranscriptSegment]:
         return words_to_segments(self.words)
 

server/uv.lock

@@ -1340,15 +1340,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9c/1f/19ebc343cc71a7ffa78f17018535adc5cbdd87afb31d7c34874680148b32/ifaddr-0.2.0-py3-none-any.whl", hash = "sha256:085e0305cfe6f16ab12d72e2024030f5d52674afad6911bb1eee207177b8a748", size = 12314, upload-time = "2022-06-15T21:40:25.756Z" },
 ]
 
-[[package]]
-name = "inflection"
-version = "0.5.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e1/7e/691d061b7329bc8d54edbf0ec22fbfb2afe61facb681f9aaa9bff7a27d04/inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", size = 15091, upload-time = "2020-08-22T08:16:29.139Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454, upload-time = "2020-08-22T08:16:27.816Z" },
-]
-
 [[package]]
 name = "iniconfig"
 version = "2.1.0"
@@ -2313,18 +2304,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/74/c1/bb7e334135859c3a92ec399bc89293ea73f28e815e35b43929c8db6af030/primePy-1.3-py3-none-any.whl", hash = "sha256:5ed443718765be9bf7e2ff4c56cdff71b42140a15b39d054f9d99f0009e2317a", size = 4040, upload-time = "2018-05-29T17:18:17.53Z" },
 ]
 
-[[package]]
-name = "profanityfilter"
-version = "2.1.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "inflection" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/8d/03/08740b5e0800f9eb9f675c149a497a3f3735e7b04e414bcce64136e7e487/profanityfilter-2.1.0.tar.gz", hash = "sha256:0ede04e92a9d7255faa52b53776518edc6586dda828aca677c74b5994dfdd9d8", size = 7910, upload-time = "2024-11-25T22:31:51.194Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/88/03/eb18f72dc6e6398e75e3762677f18ab3a773a384b18efd3ed9119844e892/profanityfilter-2.1.0-py2.py3-none-any.whl", hash = "sha256:e1bc07012760fd74512a335abb93a36877831ed26abab78bfe31bebb68f8c844", size = 7483, upload-time = "2024-11-25T22:31:50.129Z" },
-]
-
 [[package]]
 name = "prometheus-client"
 version = "0.22.1"
@@ -3133,7 +3112,6 @@ dependencies = [
     { name = "llama-index-llms-openai-like" },
     { name = "nltk" },
     { name = "openai" },
-    { name = "profanityfilter" },
     { name = "prometheus-fastapi-instrumentator" },
     { name = "protobuf" },
     { name = "psycopg2-binary" },
@@ -3210,7 +3188,6 @@ requires-dist = [
     { name = "llama-index-llms-openai-like", specifier = ">=0.4.0" },
     { name = "nltk", specifier = ">=3.8.1" },
     { name = "openai", specifier = ">=1.59.7" },
-    { name = "profanityfilter", specifier = ">=2.0.6" },
     { name = "prometheus-fastapi-instrumentator", specifier = ">=6.1.0" },
     { name = "protobuf", specifier = ">=4.24.3" },
     { name = "psycopg2-binary", specifier = ">=2.9.10" },