From 986f063857f5e623f1314b49305ed217fdfe82ed Mon Sep 17 00:00:00 2001 From: Sergey Mankovsky Date: Tue, 17 Dec 2024 11:47:29 +0100 Subject: [PATCH 1/6] Dockerize local env --- docker-compose.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9e6519af..61ae379a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,22 +7,30 @@ services: - 1250:1250 volumes: - model-cache:/root/.cache - environment: ENTRYPOINT=server + environment: + ENTRYPOINT: server worker: build: context: server volumes: - model-cache:/root/.cache - environment: ENTRYPOINT=worker + environment: + ENTRYPOINT: worker redis: image: redis:7.2 ports: - 6379:6379 web: - build: - context: www + image: node:18 ports: - - 3000:3000 + - "3000:3000" + command: sh -c "yarn install && yarn dev" + restart: unless-stopped + working_dir: /app + volumes: + - ./www:/app/ + env_file: + - ./www/.env.local volumes: model-cache: From 8d9aaeab36dbd384498c4e0da49531d4e345ed39 Mon Sep 17 00:00:00 2001 From: Sergey Mankovsky Date: Thu, 23 Jan 2025 15:30:56 +0100 Subject: [PATCH 2/6] Update compose to run reflector locally --- docker-compose.yml => compose.yml | 14 ++++++++------ server/docker-compose.yml | 27 --------------------------- www/docker-compose.yml | 11 ----------- 3 files changed, 8 insertions(+), 44 deletions(-) rename docker-compose.yml => compose.yml (78%) delete mode 100644 server/docker-compose.yml delete mode 100644 www/docker-compose.yml diff --git a/docker-compose.yml b/compose.yml similarity index 78% rename from docker-compose.yml rename to compose.yml index 61ae379a..2ba58fa0 100644 --- a/docker-compose.yml +++ b/compose.yml @@ -1,4 +1,3 @@ -version: "3.9" services: server: build: @@ -6,16 +5,22 @@ services: ports: - 1250:1250 volumes: - - model-cache:/root/.cache + - ./server/data/:/app/data/ + env_file: + - ./server/.env environment: ENTRYPOINT: server + worker: build: context: server volumes: - - model-cache:/root/.cache + - ./server/data/:/app/data/ + env_file: + - ./server/.env environment: ENTRYPOINT: worker + redis: image: redis:7.2 ports: @@ -31,6 +36,3 @@ services: - ./www:/app/ env_file: - ./www/.env.local - -volumes: - model-cache: diff --git a/server/docker-compose.yml b/server/docker-compose.yml deleted file mode 100644 index c8432816..00000000 --- a/server/docker-compose.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: "3.9" -services: - server: - build: - context: . - ports: - - 1250:1250 - volumes: - - model-cache:/root/.cache - environment: - ENTRYPOINT: server - REDIS_HOST: redis - worker: - build: - context: . - volumes: - - model-cache:/root/.cache - environment: - ENTRYPOINT: worker - REDIS_HOST: redis - redis: - image: redis:7.2 - ports: - - 6379:6379 - -volumes: - model-cache: diff --git a/www/docker-compose.yml b/www/docker-compose.yml deleted file mode 100644 index 18a4f860..00000000 --- a/www/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: "3.9" -services: - server: - build: - context: . - ports: - - 80:80 - environment: - LLM_URL: "${LLM_URL}" - volumes: - - model-cache:/root/.cache From 14d028e8c4b25f576463e56b0f5bf2575b1f9f4b Mon Sep 17 00:00:00 2001 From: Sergey Mankovsky Date: Thu, 23 Jan 2025 15:31:14 +0100 Subject: [PATCH 3/6] Remove waveform and topics on reprocessing --- server/reflector/pipelines/main_live_pipeline.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/reflector/pipelines/main_live_pipeline.py b/server/reflector/pipelines/main_live_pipeline.py index f81e37b7..d7b5ef1a 100644 --- a/server/reflector/pipelines/main_live_pipeline.py +++ b/server/reflector/pipelines/main_live_pipeline.py @@ -665,6 +665,13 @@ async def pipeline_process(transcript: Transcript, logger: Logger): try: if transcript.audio_location == "storage": await transcripts_controller.download_mp3_from_storage(transcript) + transcript.audio_waveform_filename.unlink(missing_ok=True) + await transcripts_controller.update( + transcript, + { + "topics": [], + }, + ) # open audio audio_filename = next(transcript.data_path.glob("upload.*"), None) From 4d1a22043dbbb134c00972e34d729fe78906b69b Mon Sep 17 00:00:00 2001 From: Sergey Mankovsky Date: Thu, 23 Jan 2025 17:25:07 +0100 Subject: [PATCH 4/6] Update zulip message on reprocess --- .../reflector/pipelines/main_live_pipeline.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/server/reflector/pipelines/main_live_pipeline.py b/server/reflector/pipelines/main_live_pipeline.py index d7b5ef1a..1e708c06 100644 --- a/server/reflector/pipelines/main_live_pipeline.py +++ b/server/reflector/pipelines/main_live_pipeline.py @@ -53,7 +53,11 @@ from reflector.processors.types import ( from reflector.processors.types import Transcript as TranscriptProcessorType from reflector.settings import settings from reflector.ws_manager import WebsocketManager, get_ws_manager -from reflector.zulip import get_zulip_message, send_message_to_zulip +from reflector.zulip import ( + get_zulip_message, + send_message_to_zulip, + update_zulip_message, +) from structlog import BoundLogger as Logger @@ -573,10 +577,20 @@ async def pipeline_post_to_zulip(transcript: Transcript, logger: Logger): if room.zulip_auto_post: message = get_zulip_message(transcript=transcript, include_topics=True) - response = send_message_to_zulip(room.zulip_stream, room.zulip_topic, message) - await transcripts_controller.update( - transcript, {"zulip_message_id": response["id"]} - ) + if transcript.zulip_message_id: + update_zulip_message( + transcript.zulip_message_id, + room.zulip_stream, + room.zulip_topic, + message, + ) + else: + response = send_message_to_zulip( + room.zulip_stream, room.zulip_topic, message + ) + await transcripts_controller.update( + transcript, {"zulip_message_id": response["id"]} + ) logger.info("Posted to zulip") From 87bcf9eb8d388aa8a536aec1e23387616b2552ff Mon Sep 17 00:00:00 2001 From: Sergey Mankovsky Date: Thu, 23 Jan 2025 17:37:47 +0100 Subject: [PATCH 5/6] Post new message if previous was deleted --- .../reflector/pipelines/main_live_pipeline.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/server/reflector/pipelines/main_live_pipeline.py b/server/reflector/pipelines/main_live_pipeline.py index 1e708c06..4a7aab1c 100644 --- a/server/reflector/pipelines/main_live_pipeline.py +++ b/server/reflector/pipelines/main_live_pipeline.py @@ -577,14 +577,21 @@ async def pipeline_post_to_zulip(transcript: Transcript, logger: Logger): if room.zulip_auto_post: message = get_zulip_message(transcript=transcript, include_topics=True) + message_updated = False if transcript.zulip_message_id: - update_zulip_message( - transcript.zulip_message_id, - room.zulip_stream, - room.zulip_topic, - message, - ) - else: + try: + update_zulip_message( + transcript.zulip_message_id, + room.zulip_stream, + room.zulip_topic, + message, + ) + message_updated = True + except Exception as e: + logger.error( + f"Failed to update zulip message with id {transcript.zulip_message_id}" + ) + if not message_updated: response = send_message_to_zulip( room.zulip_stream, room.zulip_topic, message ) From 983c444e3555965d781b527e760442b870528c44 Mon Sep 17 00:00:00 2001 From: Sergey Mankovsky Date: Thu, 23 Jan 2025 17:40:32 +0100 Subject: [PATCH 6/6] Remove unused var --- server/reflector/pipelines/main_live_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reflector/pipelines/main_live_pipeline.py b/server/reflector/pipelines/main_live_pipeline.py index 4a7aab1c..d54ebac0 100644 --- a/server/reflector/pipelines/main_live_pipeline.py +++ b/server/reflector/pipelines/main_live_pipeline.py @@ -587,7 +587,7 @@ async def pipeline_post_to_zulip(transcript: Transcript, logger: Logger): message, ) message_updated = True - except Exception as e: + except Exception: logger.error( f"Failed to update zulip message with id {transcript.zulip_message_id}" )