From 717b1df554b4bddeb8006ab761b56e7ecca1bb9a Mon Sep 17 00:00:00 2001 From: "Juan Diego Caballero (JDC)" Date: Tue, 18 Jul 2023 18:37:11 -0500 Subject: [PATCH 1/3] Initial --- requirements.txt | 1 + server_executor_cleaned.py | 46 ++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/requirements.txt b/requirements.txt index 76eba9d5..8b935184 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,3 +58,4 @@ httpx==0.24.1 sortedcontainers==2.4.0 https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz gpt4all==1.0.5 +aiohttp_cors diff --git a/server_executor_cleaned.py b/server_executor_cleaned.py index 2c2e54a3..4b7fccc6 100644 --- a/server_executor_cleaned.py +++ b/server_executor_cleaned.py @@ -5,6 +5,7 @@ import uuid import wave from concurrent.futures import ThreadPoolExecutor +import aiohttp_cors import jax.numpy as jnp from aiohttp import web from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription @@ -14,14 +15,12 @@ from gpt4all import GPT4All from loguru import logger from whisper_jax import FlaxWhisperPipline -from utils.run_utils import run_in_executor, config +from utils.run_utils import config, run_in_executor pcs = set() relay = MediaRelay() data_channel = None -pipeline = FlaxWhisperPipline("openai/whisper-tiny", - dtype=jnp.float16, - batch_size=16) +pipeline = FlaxWhisperPipline("openai/whisper-tiny", dtype=jnp.float16, batch_size=16) CHANNELS = 2 RATE = 48000 @@ -96,19 +95,18 @@ class AudioStreamTrack(MediaStreamTrack): audio_buffer.write(frame) if local_frames := audio_buffer.read_many(256 * 960, partial=False): whisper_result = run_in_executor( - get_transcription, local_frames, executor=executor + get_transcription, local_frames, executor=executor ) whisper_result.add_done_callback( - lambda f: channel_send(data_channel, whisper_result.result()) - if f.result() - else None + lambda f: channel_send(data_channel, whisper_result.result()) + if f.result() + else None ) - llm_result = run_in_executor(get_title_and_summary, - executor=executor) + llm_result = run_in_executor(get_title_and_summary, executor=executor) llm_result.add_done_callback( - lambda f: channel_send(data_channel, llm_result.result()) - if f.result() - else None + lambda f: channel_send(data_channel, llm_result.result()) + if f.result() + else None ) return frame @@ -156,11 +154,10 @@ async def offer(request): answer = await pc.createAnswer() await pc.setLocalDescription(answer) return web.Response( - content_type="application/json", - text=json.dumps( - {"sdp": pc.localDescription.sdp, - "type": pc.localDescription.type} - ), + content_type="application/json", + text=json.dumps( + {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} + ), ) @@ -172,6 +169,17 @@ async def on_shutdown(app): if __name__ == "__main__": app = web.Application() + + cors = aiohttp_cors.setup( + app, + defaults={ + "*": aiohttp_cors.ResourceOptions( + allow_credentials=True, expose_headers="*", allow_headers="*" + ) + }, + ) + + offer_resource = cors.add(app.router.add_resource("/offer")) + cors.add(offer_resource.add_route("POST", offer)) app.on_shutdown.append(on_shutdown) - app.router.add_post("/offer", offer) web.run_app(app, access_log=None, host="127.0.0.1", port=1250) From 0c4d3f8b0d3f9bf66a5debd72188110dbd71f423 Mon Sep 17 00:00:00 2001 From: "Juan Diego Caballero (JDC)" Date: Tue, 18 Jul 2023 18:52:23 -0500 Subject: [PATCH 2/3] Fix Bug --- server_executor_cleaned.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server_executor_cleaned.py b/server_executor_cleaned.py index 4b7fccc6..03de3008 100644 --- a/server_executor_cleaned.py +++ b/server_executor_cleaned.py @@ -58,7 +58,7 @@ def channel_log(channel, t, message): def channel_send(channel, message): # channel_log(channel, ">", message) if channel and message: - channel.send(str(message)) + channel.send(json.dumps(message)) def get_transcription(frames): From 94ac5ece322698211fc197371df81a1a032b486f Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Tue, 25 Jul 2023 14:07:26 +0530 Subject: [PATCH 3/3] merge with main and resolve conflicts --- pipeline-requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipeline-requirements.txt b/pipeline-requirements.txt index 6b46e1b3..9a0ffc57 100644 --- a/pipeline-requirements.txt +++ b/pipeline-requirements.txt @@ -54,4 +54,8 @@ stamina==23.1.0 httpx==0.24.1 https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz gpt4all==1.0.5 -aiohttp_cors +aiohttp==3.8.5 +aiohttp-cors==0.7.0 +aioice==0.9.0 +aiortc==1.5.0 +aiosignal==1.3.1 \ No newline at end of file