Merge branch 'main' into mathieu/calendar-integration-rebased

2025-12-21 04:39:06 +00:00 · 2025-09-16 15:53:18 -04:00
parent 1087ce5b3a fa049e8d06
commit 5d0a571247
9 changed files with 108 additions and 114 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,18 @@
 # Changelog

+## [0.11.0](https://github.com/Monadical-SAS/reflector/compare/v0.10.0...v0.11.0) (2025-09-16)
+
+
+### Features
+
+* remove profanity filter that was there for conference ([#652](https://github.com/Monadical-SAS/reflector/issues/652)) ([b42f7cf](https://github.com/Monadical-SAS/reflector/commit/b42f7cfc606783afcee792590efcc78b507468ab))
+
+
+### Bug Fixes
+
+* zulip and consent handler on the file pipeline ([#645](https://github.com/Monadical-SAS/reflector/issues/645)) ([5f143fe](https://github.com/Monadical-SAS/reflector/commit/5f143fe3640875dcb56c26694254a93189281d17))
+* zulip stream and topic selection in share dialog ([#644](https://github.com/Monadical-SAS/reflector/issues/644)) ([c546e69](https://github.com/Monadical-SAS/reflector/commit/c546e69739e68bb74fbc877eb62609928e5b8de6))
+
 ## [0.10.0](https://github.com/Monadical-SAS/reflector/compare/v0.9.0...v0.10.0) (2025-09-11)


--- a/server/pyproject.toml
+++ b/server/pyproject.toml
@@ -26,7 +26,6 @@ dependencies = [
    "prometheus-fastapi-instrumentator>=6.1.0",
    "sentencepiece>=0.1.99",
    "protobuf>=4.24.3",
-    "profanityfilter>=2.0.6",
    "celery>=5.3.4",
    "redis>=5.0.1",
    "python-jose[cryptography]>=3.3.0",
--- a/server/reflector/pipelines/main_file_pipeline.py
+++ b/server/reflector/pipelines/main_file_pipeline.py
@@ -12,7 +12,7 @@ from pathlib import Path

 import av
 import structlog
-from celery import shared_task
+from celery import chain, shared_task

 from reflector.asynctask import asynctask
 from reflector.db.rooms import rooms_controller
@@ -26,6 +26,8 @@ from reflector.logger import logger
 from reflector.pipelines.main_live_pipeline import (
    PipelineMainBase,
    broadcast_to_sockets,
+    task_cleanup_consent,
+    task_pipeline_post_to_zulip,
 )
 from reflector.processors import (
    AudioFileWriterProcessor,
@@ -379,6 +381,28 @@ class PipelineMainFile(PipelineMainBase):
        await processor.flush()


+@shared_task
+@asynctask
+async def task_send_webhook_if_needed(*, transcript_id: str):
+    """Send webhook if this is a room recording with webhook configured"""
+    transcript = await transcripts_controller.get_by_id(transcript_id)
+    if not transcript:
+        return
+
+    if transcript.source_kind == SourceKind.ROOM and transcript.room_id:
+        room = await rooms_controller.get_by_id(transcript.room_id)
+        if room and room.webhook_url:
+            logger.info(
+                "Dispatching webhook",
+                transcript_id=transcript_id,
+                room_id=room.id,
+                webhook_url=room.webhook_url,
+            )
+            send_transcript_webhook.delay(
+                transcript_id, room.id, event_id=uuid.uuid4().hex
+            )
+
+
@shared_task
@asynctask
 async def task_pipeline_file_process(*, transcript_id: str):
@@ -406,16 +430,10 @@ async def task_pipeline_file_process(*, transcript_id: str):
        await pipeline.set_status(transcript_id, "error")
        raise

-    # Trigger webhook if this is a room recording with webhook configured
-    if transcript.source_kind == SourceKind.ROOM and transcript.room_id:
-        room = await rooms_controller.get_by_id(transcript.room_id)
-        if room and room.webhook_url:
-            logger.info(
-                "Dispatching webhook task",
-                transcript_id=transcript_id,
-                room_id=room.id,
-                webhook_url=room.webhook_url,
-            )
-            send_transcript_webhook.delay(
-                transcript_id, room.id, event_id=uuid.uuid4().hex
+    # Run post-processing chain: consent cleanup -> zulip -> webhook
+    post_chain = chain(
+        task_cleanup_consent.si(transcript_id=transcript_id),
+        task_pipeline_post_to_zulip.si(transcript_id=transcript_id),
+        task_send_webhook_if_needed.si(transcript_id=transcript_id),
    )
+    post_chain.delay()
--- a/server/reflector/processors/types.py
+++ b/server/reflector/processors/types.py
@@ -4,11 +4,8 @@ import tempfile
 from pathlib import Path
 from typing import Annotated, TypedDict

-from profanityfilter import ProfanityFilter
 from pydantic import BaseModel, Field, PrivateAttr

-from reflector.redis_cache import redis_cache
-

 class DiarizationSegment(TypedDict):
    """Type definition for diarization segment containing speaker information"""
@@ -20,9 +17,6 @@ class DiarizationSegment(TypedDict):

 PUNC_RE = re.compile(r"[.;:?!…]")

-profanity_filter = ProfanityFilter()
-profanity_filter.set_censor("*")
-

 class AudioFile(BaseModel):
    name: str
@@ -124,21 +118,11 @@ def words_to_segments(words: list[Word]) -> list[TranscriptSegment]:

 class Transcript(BaseModel):
    translation: str | None = None
-    words: list[Word] = None
-
-    @property
-    def raw_text(self):
-        # Uncensored text
-        return "".join([word.text for word in self.words])
-
-    @redis_cache(prefix="profanity", duration=3600 * 24 * 7)
-    def _get_censored_text(self, text: str):
-        return profanity_filter.censor(text).strip()
+    words: list[Word] = []

    @property
    def text(self):
-        # Censored text
-        return self._get_censored_text(self.raw_text)
+        return "".join([word.text for word in self.words])

    @property
    def human_timestamp(self):
@@ -170,12 +154,6 @@ class Transcript(BaseModel):
            word.start += offset
            word.end += offset

-    def clone(self):
-        words = [
-            Word(text=word.text, start=word.start, end=word.end) for word in self.words
-        ]
-        return Transcript(text=self.text, translation=self.translation, words=words)
-
    def as_segments(self) -> list[TranscriptSegment]:
        return words_to_segments(self.words)

--- a/server/uv.lock
+++ b/server/uv.lock
@@ -1340,15 +1340,6 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/9c/1f/19ebc343cc71a7ffa78f17018535adc5cbdd87afb31d7c34874680148b32/ifaddr-0.2.0-py3-none-any.whl", hash = "sha256:085e0305cfe6f16ab12d72e2024030f5d52674afad6911bb1eee207177b8a748", size = 12314, upload-time = "2022-06-15T21:40:25.756Z" },
 ]

-[[package]]
-name = "inflection"
-version = "0.5.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e1/7e/691d061b7329bc8d54edbf0ec22fbfb2afe61facb681f9aaa9bff7a27d04/inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", size = 15091, upload-time = "2020-08-22T08:16:29.139Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454, upload-time = "2020-08-22T08:16:27.816Z" },
-]
-
 [[package]]
 name = "iniconfig"
 version = "2.1.0"
@@ -2313,18 +2304,6 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/74/c1/bb7e334135859c3a92ec399bc89293ea73f28e815e35b43929c8db6af030/primePy-1.3-py3-none-any.whl", hash = "sha256:5ed443718765be9bf7e2ff4c56cdff71b42140a15b39d054f9d99f0009e2317a", size = 4040, upload-time = "2018-05-29T17:18:17.53Z" },
 ]

-[[package]]
-name = "profanityfilter"
-version = "2.1.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "inflection" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/8d/03/08740b5e0800f9eb9f675c149a497a3f3735e7b04e414bcce64136e7e487/profanityfilter-2.1.0.tar.gz", hash = "sha256:0ede04e92a9d7255faa52b53776518edc6586dda828aca677c74b5994dfdd9d8", size = 7910, upload-time = "2024-11-25T22:31:51.194Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/88/03/eb18f72dc6e6398e75e3762677f18ab3a773a384b18efd3ed9119844e892/profanityfilter-2.1.0-py2.py3-none-any.whl", hash = "sha256:e1bc07012760fd74512a335abb93a36877831ed26abab78bfe31bebb68f8c844", size = 7483, upload-time = "2024-11-25T22:31:50.129Z" },
-]
-
 [[package]]
 name = "prometheus-client"
 version = "0.22.1"
@@ -3133,7 +3112,6 @@ dependencies = [
    { name = "llama-index-llms-openai-like" },
    { name = "nltk" },
    { name = "openai" },
-    { name = "profanityfilter" },
    { name = "prometheus-fastapi-instrumentator" },
    { name = "protobuf" },
    { name = "psycopg2-binary" },
@@ -3210,7 +3188,6 @@ requires-dist = [
    { name = "llama-index-llms-openai-like", specifier = ">=0.4.0" },
    { name = "nltk", specifier = ">=3.8.1" },
    { name = "openai", specifier = ">=1.59.7" },
-    { name = "profanityfilter", specifier = ">=2.0.6" },
    { name = "prometheus-fastapi-instrumentator", specifier = ">=6.1.0" },
    { name = "protobuf", specifier = ">=4.24.3" },
    { name = "psycopg2-binary", specifier = ">=2.9.10" },
--- a/www/app/(app)/transcripts/player.tsx
+++ b/www/app/(app)/transcripts/player.tsx
@@ -33,17 +33,27 @@ export default function Player(props: PlayerProps) {
  const topicsRef = useRef(props.topics);
  const [firstRender, setFirstRender] = useState<boolean>(true);

-  const keyHandler = (e) => {
-    if (e.key == " ") {
+  const shouldIgnoreHotkeys = (target: EventTarget | null) => {
+    return (
+      target instanceof HTMLInputElement ||
+      target instanceof HTMLTextAreaElement
+    );
+  };
+
+  const keyHandler = (e: KeyboardEvent) => {
+    if (e.key === " ") {
+      if (e.repeat) return;
+      if (shouldIgnoreHotkeys(e.target)) return;
+      e.preventDefault();
      wavesurfer?.playPause();
    }
  };
  useEffect(() => {
-    document.addEventListener("keyup", keyHandler);
+    document.addEventListener("keydown", keyHandler);
    return () => {
-      document.removeEventListener("keyup", keyHandler);
+      document.removeEventListener("keydown", keyHandler);
    };
-  });
+  }, [wavesurfer]);

  // Waveform setup
  useEffect(() => {
--- a/www/app/(app)/transcripts/shareZulip.tsx
+++ b/www/app/(app)/transcripts/shareZulip.tsx
@@ -14,8 +14,7 @@ import {
  Checkbox,
  Combobox,
  Spinner,
-  useFilter,
-  useListCollection,
+  createListCollection,
 } from "@chakra-ui/react";
 import { TbBrandZulip } from "react-icons/tb";
 import {
@@ -48,8 +47,6 @@ export default function ShareZulip(props: ShareZulipProps & BoxProps) {
  const { data: topics = [] } = useZulipTopics(selectedStreamId);
  const postToZulipMutation = useTranscriptPostToZulip();

-  const { contains } = useFilter({ sensitivity: "base" });
-
  const streamItems = useMemo(() => {
    return streams.map((stream: Stream) => ({
      label: stream.name,
@@ -64,17 +61,21 @@ export default function ShareZulip(props: ShareZulipProps & BoxProps) {
    }));
  }, [topics]);

-  const { collection: streamItemsCollection, filter: streamItemsFilter } =
-    useListCollection({
-      initialItems: streamItems,
-      filter: contains,
-    });
+  const streamCollection = useMemo(
+    () =>
+      createListCollection({
+        items: streamItems,
+      }),
+    [streamItems],
+  );

-  const { collection: topicItemsCollection, filter: topicItemsFilter } =
-    useListCollection({
-      initialItems: topicItems,
-      filter: contains,
-    });
+  const topicCollection = useMemo(
+    () =>
+      createListCollection({
+        items: topicItems,
+      }),
+    [topicItems],
+  );

  // Update selected stream ID when stream changes
  useEffect(() => {
@@ -156,15 +157,12 @@ export default function ShareZulip(props: ShareZulipProps & BoxProps) {
                    <Flex align="center" gap={2}>
                      <Text>#</Text>
                      <Combobox.Root
-                        collection={streamItemsCollection}
+                        collection={streamCollection}
                        value={stream ? [stream] : []}
                        onValueChange={(e) => {
                          setTopic(undefined);
                          setStream(e.value[0]);
                        }}
-                        onInputValueChange={(e) =>
-                          streamItemsFilter(e.inputValue)
-                        }
                        openOnClick={true}
                        positioning={{
                          strategy: "fixed",
@@ -181,7 +179,7 @@ export default function ShareZulip(props: ShareZulipProps & BoxProps) {
                        <Combobox.Positioner>
                          <Combobox.Content>
                            <Combobox.Empty>No streams found</Combobox.Empty>
-                            {streamItemsCollection.items.map((item) => (
+                            {streamItems.map((item) => (
                              <Combobox.Item key={item.value} item={item}>
                                {item.label}
                              </Combobox.Item>
@@ -197,12 +195,9 @@ export default function ShareZulip(props: ShareZulipProps & BoxProps) {
                      <Flex align="center" gap={2}>
                        <Text visibility="hidden">#</Text>
                        <Combobox.Root
-                          collection={topicItemsCollection}
+                          collection={topicCollection}
                          value={topic ? [topic] : []}
                          onValueChange={(e) => setTopic(e.value[0])}
-                          onInputValueChange={(e) =>
-                            topicItemsFilter(e.inputValue)
-                          }
                          openOnClick
                          selectionBehavior="replace"
                          skipAnimationOnMount={true}
@@ -222,7 +217,7 @@ export default function ShareZulip(props: ShareZulipProps & BoxProps) {
                          <Combobox.Positioner>
                            <Combobox.Content>
                              <Combobox.Empty>No topics found</Combobox.Empty>
-                              {topicItemsCollection.items.map((item) => (
+                              {topicItems.map((item) => (
                                <Combobox.Item key={item.value} item={item}>
                                  {item.label}
                                  <Combobox.ItemIndicator />
--- a/www/app/lib/apiClient.tsx
+++ b/www/app/lib/apiClient.tsx
@@ -3,8 +3,10 @@
 import createClient from "openapi-fetch";
 import type { paths } from "../reflector-api";
 import createFetchClient from "openapi-react-query";
-import { assertExistsAndNonEmptyString } from "./utils";
+import { assertExistsAndNonEmptyString, parseNonEmptyString } from "./utils";
 import { isBuildPhase } from "./next";
+import { getSession } from "next-auth/react";
+import { assertExtendedToken } from "./types";

 export const API_URL = !isBuildPhase
  ? assertExistsAndNonEmptyString(
@@ -21,29 +23,29 @@ export const client = createClient<paths>({
  baseUrl: API_URL,
 });

-const waitForAuthTokenDefinitivePresenceOrAbscence = async () => {
-  let tries = 0;
-  let time = 0;
-  const STEP = 100;
-  while (currentAuthToken === undefined) {
-    await new Promise((resolve) => setTimeout(resolve, STEP));
-    time += STEP;
-    tries++;
-    // most likely first try is more than enough, if it's more there's already something weird happens
-    if (tries > 10) {
-      // even when there's no auth assumed at all, we probably should explicitly call configureApiAuth(null)
-      throw new Error(
-        `Could not get auth token definitive presence/absence in ${time}ms. not calling configureApiAuth?`,
-      );
-    }
+// will assert presence/absence of login initially
+const initialSessionPromise = getSession();
+
+const waitForAuthTokenDefinitivePresenceOrAbsence = async () => {
+  const initialSession = await initialSessionPromise;
+  if (currentAuthToken === undefined) {
+    currentAuthToken =
+      initialSession === null
+        ? null
+        : assertExtendedToken(initialSession).accessToken;
  }
+  // otherwise already overwritten by external forces
+  return currentAuthToken;
 };

 client.use({
  async onRequest({ request }) {
-    await waitForAuthTokenDefinitivePresenceOrAbscence();
-    if (currentAuthToken) {
-      request.headers.set("Authorization", `Bearer ${currentAuthToken}`);
+    const token = await waitForAuthTokenDefinitivePresenceOrAbsence();
+    if (token !== null) {
+      request.headers.set(
+        "Authorization",
+        `Bearer ${parseNonEmptyString(token)}`,
+      );
    }
    // XXX Only set Content-Type if not already set (FormData will set its own boundary)
    // This is a work around for uploading file, we're passing a formdata
--- a/www/app/lib/types.ts
+++ b/www/app/lib/types.ts
@@ -21,7 +21,7 @@ export interface CustomSession extends Session {
 // assumption that JWT is JWTWithAccessToken - we set it in jwt callback of auth; typing isn't strong around there
 // but the assumption is crucial to auth working
 export const assertExtendedToken = <T>(
-  t: T,
+  t: Exclude<T, null | undefined>,
 ): T & {
  accessTokenExpires: number;
  accessToken: string;
@@ -45,7 +45,7 @@ export const assertExtendedToken = <T>(
 };

 export const assertExtendedTokenAndUserId = <U, T extends { user?: U }>(
-  t: T,
+  t: Exclude<T, null | undefined>,
 ): T & {
  accessTokenExpires: number;
  accessToken: string;
@@ -55,7 +55,7 @@ export const assertExtendedTokenAndUserId = <U, T extends { user?: U }>(
 } => {
  const extendedToken = assertExtendedToken(t);
  if (typeof (extendedToken.user as any)?.id === "string") {
-    return t as T & {
+    return t as Exclude<T, null | undefined> & {
      accessTokenExpires: number;
      accessToken: string;
      user: U & {
@@ -67,7 +67,9 @@ export const assertExtendedTokenAndUserId = <U, T extends { user?: U }>(
 };

 // best attempt to check the session is valid
-export const assertCustomSession = <S extends Session>(s: S): CustomSession => {
+export const assertCustomSession = <T extends Session>(
+  s: Exclude<T, null | undefined>,
+): CustomSession => {
  const r = assertExtendedTokenAndUserId(s);
  // no other checks for now
  return r as CustomSession;