From 73327d2e9e0dbe0d53b95025292aeff7bbd34339 Mon Sep 17 00:00:00 2001 From: Sara Date: Thu, 2 Nov 2023 18:49:37 +0100 Subject: [PATCH 01/30] adds timeout and humanMessage --- www/app/(errors)/errorContext.tsx | 12 ++++- www/app/(errors)/errorMessage.tsx | 52 ++++++++++++++++--- www/app/[domain]/browse/pagination.tsx | 4 +- .../[domain]/transcripts/createTranscript.ts | 5 +- www/app/[domain]/transcripts/recorder.tsx | 5 -- www/app/[domain]/transcripts/useMp3.ts | 2 +- www/app/[domain]/transcripts/useTopics.ts | 2 +- www/app/[domain]/transcripts/useTranscript.ts | 2 +- www/app/[domain]/transcripts/useWaveform.ts | 2 +- www/app/[domain]/transcripts/useWebRTC.ts | 4 +- www/app/[domain]/transcripts/useWebSockets.ts | 2 +- 11 files changed, 68 insertions(+), 24 deletions(-) diff --git a/www/app/(errors)/errorContext.tsx b/www/app/(errors)/errorContext.tsx index d8a80c04..d541c6f0 100644 --- a/www/app/(errors)/errorContext.tsx +++ b/www/app/(errors)/errorContext.tsx @@ -3,7 +3,8 @@ import React, { createContext, useContext, useState } from "react"; interface ErrorContextProps { error: Error | null; - setError: React.Dispatch>; + humanMessage?: string; + setError: (error: Error, humanMessage?: string) => void; } const ErrorContext = createContext(undefined); @@ -22,9 +23,16 @@ interface ErrorProviderProps { export const ErrorProvider: React.FC = ({ children }) => { const [error, setError] = useState(null); + const [humanMessage, setHumanMessage] = useState(); + const declareError = (error, humanMessage?) => { + setError(error); + setHumanMessage(humanMessage); + }; return ( - + {children} ); diff --git a/www/app/(errors)/errorMessage.tsx b/www/app/(errors)/errorMessage.tsx index 8b410c4c..8489af92 100644 --- a/www/app/(errors)/errorMessage.tsx +++ b/www/app/(errors)/errorMessage.tsx @@ -4,29 +4,67 @@ import { useEffect, useState } from "react"; import * as Sentry from "@sentry/react"; const ErrorMessage: React.FC = () => { - const { error, setError } = useError(); + const { error, setError, humanMessage } = useError(); const [isVisible, setIsVisible] = useState(false); + const [timeoutId, setTimeoutId] = useState(); + + // Setup Shortcuts + useEffect(() => { + const handleKeyPress = (event: KeyboardEvent) => { + switch (event.key) { + case "^": + throw new Error("Unhandled Exception thrown by '^' shortcut"); + case "$": + setError( + new Error("Unhandled Exception thrown by '$' shortcut"), + "You did this to yourself", + ); + } + }; + + document.addEventListener("keydown", handleKeyPress); + return () => document.removeEventListener("keydown", handleKeyPress); + }, []); useEffect(() => { if (error) { - setIsVisible(true); - Sentry.captureException(error); - console.error("Error", error.message, error); + if (humanMessage) { + setIsVisible(true); + Sentry.captureException(Error(humanMessage, { cause: error })); + } else { + Sentry.captureException(error); + } + + console.error("Error", error); } }, [error]); - if (!isVisible || !error) return null; + useEffect(() => { + if (isVisible) { + setTimeoutId( + setTimeout(() => { + setIsVisible(false); + setTimeoutId(undefined); + }, 30000), + ); + } + if (!isVisible && timeoutId) { + clearTimeout(timeoutId); + } + }, [isVisible]); + + if (!isVisible || !humanMessage) return null; return ( ); }; diff --git a/www/app/[domain]/browse/pagination.tsx b/www/app/[domain]/browse/pagination.tsx index 27ff5f47..8716b634 100644 --- a/www/app/[domain]/browse/pagination.tsx +++ b/www/app/[domain]/browse/pagination.tsx @@ -52,7 +52,7 @@ export default function Pagination(props: PaginationProps) { {pageNumbers.map((pageNumber) => ( + + {props.audioDevices && props.audioDevices?.length > 0 && deviceId && ( <> - - + {!isRecording && ( + + )} {props.audioDevices && props.audioDevices?.length > 0 && deviceId && ( <> + + ); +} diff --git a/www/app/[domain]/transcripts/recorder.tsx b/www/app/[domain]/transcripts/recorder.tsx index 8db32ff7..5b4420c4 100644 --- a/www/app/[domain]/transcripts/recorder.tsx +++ b/www/app/[domain]/transcripts/recorder.tsx @@ -6,11 +6,8 @@ import CustomRegionsPlugin from "../../lib/custom-plugins/regions"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faMicrophone } from "@fortawesome/free-solid-svg-icons"; -import { faDownload } from "@fortawesome/free-solid-svg-icons"; import { formatTime } from "../../lib/time"; -import { Topic } from "./webSocketTypes"; -import { AudioWaveform } from "../../api"; import AudioInputsDropdown from "./audioInputsDropdown"; import { Option } from "react-dropdown"; import { waveSurferStyles } from "../../styles/recorder"; @@ -19,17 +16,8 @@ import { useError } from "../../(errors)/errorContext"; type RecorderProps = { setStream?: React.Dispatch>; onStop?: () => void; - topics: Topic[]; getAudioStream?: (deviceId) => Promise; audioDevices?: Option[]; - useActiveTopic: [ - Topic | null, - React.Dispatch>, - ]; - waveform?: AudioWaveform | null; - isPastMeeting: boolean; - transcriptId?: string | null; - media?: HTMLMediaElement | null; mediaDuration?: number | null; }; @@ -38,7 +26,7 @@ export default function Recorder(props: RecorderProps) { const [wavesurfer, setWavesurfer] = useState(null); const [record, setRecord] = useState(null); const [isRecording, setIsRecording] = useState(false); - const [hasRecorded, setHasRecorded] = useState(props.isPastMeeting); + const [hasRecorded, setHasRecorded] = useState(false); const [isPlaying, setIsPlaying] = useState(false); const [currentTime, setCurrentTime] = useState(0); const [timeInterval, setTimeInterval] = useState(null); @@ -48,8 +36,6 @@ export default function Recorder(props: RecorderProps) { ); const [deviceId, setDeviceId] = useState(null); const [recordStarted, setRecordStarted] = useState(false); - const [activeTopic, setActiveTopic] = props.useActiveTopic; - const topicsRef = useRef(props.topics); const [showDevices, setShowDevices] = useState(false); const { setError } = useError(); @@ -73,8 +59,6 @@ export default function Recorder(props: RecorderProps) { if (!record.isRecording()) return; handleRecClick(); break; - case "^": - throw new Error("Unhandled Exception thrown by '^' shortcut"); case "(": location.href = "/login"; break; @@ -104,14 +88,8 @@ export default function Recorder(props: RecorderProps) { // Waveform setup useEffect(() => { if (waveformRef.current) { - // XXX duration is required to prevent recomputing peaks from audio - // However, the current waveform returns only the peaks, and no duration - // And the backend does not save duration properly. - // So at the moment, we deduct the duration from the topics. - // This is not ideal, but it works for now. const _wavesurfer = WaveSurfer.create({ container: waveformRef.current, - peaks: props.waveform?.data, hideScrollbar: true, autoCenter: true, barWidth: 2, @@ -121,10 +99,8 @@ export default function Recorder(props: RecorderProps) { ...waveSurferStyles.player, }); - if (!props.transcriptId) { - const _wshack: any = _wavesurfer; - _wshack.renderer.renderSingleCanvas = () => {}; - } + const _wshack: any = _wavesurfer; + _wshack.renderer.renderSingleCanvas = () => {}; // styling const wsWrapper = _wavesurfer.getWrapper(); @@ -144,12 +120,6 @@ export default function Recorder(props: RecorderProps) { setRecord(_wavesurfer.registerPlugin(RecordPlugin.create())); setWaveRegions(_wavesurfer.registerPlugin(CustomRegionsPlugin.create())); - if (props.isPastMeeting) _wavesurfer.toggleInteraction(true); - - if (props.media) { - _wavesurfer.setMediaElement(props.media); - } - setWavesurfer(_wavesurfer); return () => { @@ -161,58 +131,6 @@ export default function Recorder(props: RecorderProps) { } }, []); - useEffect(() => { - if (!wavesurfer) return; - if (!props.media) return; - wavesurfer.setMediaElement(props.media); - }, [props.media, wavesurfer]); - - useEffect(() => { - topicsRef.current = props.topics; - if (!isRecording) renderMarkers(); - }, [props.topics, waveRegions]); - - const renderMarkers = () => { - if (!waveRegions) return; - - waveRegions.clearRegions(); - - for (let topic of topicsRef.current) { - const content = document.createElement("div"); - content.setAttribute("style", waveSurferStyles.marker); - content.onmouseover = () => { - content.style.backgroundColor = - waveSurferStyles.markerHover.backgroundColor; - content.style.zIndex = "999"; - content.style.width = "300px"; - }; - content.onmouseout = () => { - content.setAttribute("style", waveSurferStyles.marker); - }; - content.textContent = topic.title; - - const region = waveRegions.addRegion({ - start: topic.timestamp, - content, - color: "f00", - drag: false, - }); - region.on("click", (e) => { - e.stopPropagation(); - setActiveTopic(topic); - wavesurfer?.setTime(region.start); - }); - } - }; - - useEffect(() => { - if (!record) return; - - return record.on("stopRecording", () => { - renderMarkers(); - }); - }, [record]); - useEffect(() => { if (isRecording) { const interval = window.setInterval(() => { @@ -229,12 +147,6 @@ export default function Recorder(props: RecorderProps) { } }, [isRecording]); - useEffect(() => { - if (activeTopic) { - wavesurfer?.setTime(activeTopic.timestamp); - } - }, [activeTopic]); - const handleRecClick = async () => { if (!record) return console.log("no record"); @@ -320,7 +232,6 @@ export default function Recorder(props: RecorderProps) { if (!record) return; if (!destinationStream) return; if (props.setStream) props.setStream(destinationStream); - waveRegions?.clearRegions(); if (destinationStream) { record.startRecording(destinationStream); setIsRecording(true); @@ -379,23 +290,9 @@ export default function Recorder(props: RecorderProps) { } text-white ml-2 md:ml:4 md:h-[78px] md:min-w-[100px] text-lg`} id="play-btn" onClick={handlePlayClick} - disabled={isRecording} > {isPlaying ? "Pause" : "Play"} - - {props.transcriptId && ( - - - - )} )} {!hasRecorded && ( From 14ebfa53a8c2f9abad2342ebe7d3ab4a018e52f5 Mon Sep 17 00:00:00 2001 From: Sara Date: Mon, 13 Nov 2023 17:33:12 +0100 Subject: [PATCH 19/30] wip loading and redirects --- .../transcripts/[transcriptId]/page.tsx | 38 +++++++++++++++---- .../[transcriptId]/record/page.tsx | 26 ++++++++----- www/app/[domain]/transcripts/useTranscript.ts | 27 ++++++++++--- www/app/[domain]/transcripts/useWebSockets.ts | 30 +++++++++------ 4 files changed, 87 insertions(+), 34 deletions(-) diff --git a/www/app/[domain]/transcripts/[transcriptId]/page.tsx b/www/app/[domain]/transcripts/[transcriptId]/page.tsx index bebfe261..add5a824 100644 --- a/www/app/[domain]/transcripts/[transcriptId]/page.tsx +++ b/www/app/[domain]/transcripts/[transcriptId]/page.tsx @@ -6,7 +6,7 @@ import useWaveform from "../useWaveform"; import useMp3 from "../useMp3"; import { TopicList } from "../topicList"; import { Topic } from "../webSocketTypes"; -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import "../../../styles/button.css"; import FinalSummary from "../finalSummary"; import ShareLink from "../shareLink"; @@ -31,7 +31,7 @@ export default function TranscriptDetails(details: TranscriptDetails) { const useActiveTopic = useState(null); const mp3 = useMp3(protectedPath, transcriptId); - if (transcript?.error /** || topics?.error || waveform?.error **/) { + if (transcript?.error || topics?.error) { return ( { + const statusToRedirect = ["idle", "recording", "processing"]; + if (statusToRedirect.includes(transcript.response?.status)) { + const newUrl = "/transcripts/" + details.params.transcriptId + "/record"; + // Shallow redirection does not work on NextJS 13 + // https://github.com/vercel/next.js/discussions/48110 + // https://github.com/vercel/next.js/discussions/49540 + // router.push(newUrl, undefined, { shallow: true }); + history.replaceState({}, "", newUrl); + } + }, [transcript.response?.status]); + const fullTranscript = topics.topics ?.map((topic) => topic.transcript) .join("\n\n") .replace(/ +/g, " ") .trim() || ""; + console.log("calf full transcript"); return ( <> - {!transcriptId || transcript?.loading || topics?.loading ? ( + {transcript?.loading || topics?.loading ? ( ) : ( <> @@ -61,7 +74,7 @@ export default function TranscriptDetails(details: TranscriptDetails) { transcriptId={transcript.response.id} /> )} - {!waveform?.loading && ( + {waveform.waveform && mp3.media ? ( + ) : mp3.error || waveform.error ? ( + "error loading this recording" + ) : ( + "Loading Recording" )}
+
- {transcript?.response?.longSummary && ( + {transcript.response.longSummary ? ( + ) : transcript.response.status == "processing" ? ( + "Loading Transcript" + ) : ( + "error final summary" )}
diff --git a/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx b/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx index 41a2d053..10297f5c 100644 --- a/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx +++ b/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx @@ -8,12 +8,12 @@ import { useWebSockets } from "../../useWebSockets"; import useAudioDevice from "../../useAudioDevice"; import "../../../../styles/button.css"; import { Topic } from "../../webSocketTypes"; -import getApi from "../../../../lib/getApi"; import LiveTrancription from "../../liveTranscription"; import DisconnectedIndicator from "../../disconnectedIndicator"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faGear } from "@fortawesome/free-solid-svg-icons"; import { lockWakeState, releaseWakeState } from "../../../../lib/wakeLock"; +import { useRouter } from "next/navigation"; type TranscriptDetails = { params: { @@ -45,21 +45,31 @@ const TranscriptRecord = (details: TranscriptDetails) => { const [hasRecorded, setHasRecorded] = useState(false); const [transcriptStarted, setTranscriptStarted] = useState(false); + const router = useRouter(); + useEffect(() => { if (!transcriptStarted && webSockets.transcriptText.length !== 0) setTranscriptStarted(true); }, [webSockets.transcriptText]); useEffect(() => { - if (transcript?.response?.longSummary) { - const newUrl = `/transcripts/${transcript.response.id}`; + const statusToRedirect = ["ended", "error"]; + console.log(webSockets.status, "hey"); + console.log(transcript.response, "ho"); + + //TODO if has no topic and is error, get back to new + if ( + statusToRedirect.includes(transcript.response?.status) || + statusToRedirect.includes(webSockets.status.value) + ) { + const newUrl = "/transcripts/" + details.params.transcriptId; // Shallow redirection does not work on NextJS 13 // https://github.com/vercel/next.js/discussions/48110 // https://github.com/vercel/next.js/discussions/49540 - // router.push(newUrl, undefined, { shallow: true }); - history.replaceState({}, "", newUrl); + router.push(newUrl, undefined); + // history.replaceState({}, "", newUrl); } - }); + }, [webSockets.status.value, transcript.response?.status]); useEffect(() => { lockWakeState(); @@ -77,10 +87,7 @@ const TranscriptRecord = (details: TranscriptDetails) => { setHasRecorded(true); webRTC?.send(JSON.stringify({ cmd: "STOP" })); }} - topics={webSockets.topics} getAudioStream={getAudioStream} - useActiveTopic={useActiveTopic} - isPastMeeting={false} audioDevices={audioDevices} /> @@ -128,6 +135,7 @@ const TranscriptRecord = (details: TranscriptDetails) => { couple of minutes. Please do not navigate away from the page during this time.

+ {/* TODO If login required remove last sentence */}
)} diff --git a/www/app/[domain]/transcripts/useTranscript.ts b/www/app/[domain]/transcripts/useTranscript.ts index af60cd3b..987e57f3 100644 --- a/www/app/[domain]/transcripts/useTranscript.ts +++ b/www/app/[domain]/transcripts/useTranscript.ts @@ -5,16 +5,28 @@ import { useError } from "../../(errors)/errorContext"; import getApi from "../../lib/getApi"; import { shouldShowError } from "../../lib/errorUtils"; -type Transcript = { - response: GetTranscript | null; - loading: boolean; - error: Error | null; +type ErrorTranscript = { + error: Error; + loading: false; + response: any; +}; + +type LoadingTranscript = { + response: any; + loading: true; + error: false; +}; + +type SuccessTranscript = { + response: GetTranscript; + loading: false; + error: null; }; const useTranscript = ( protectedPath: boolean, id: string | null, -): Transcript => { +): ErrorTranscript | LoadingTranscript | SuccessTranscript => { const [response, setResponse] = useState(null); const [loading, setLoading] = useState(true); const [error, setErrorState] = useState(null); @@ -46,7 +58,10 @@ const useTranscript = ( }); }, [id, !api]); - return { response, loading, error }; + return { response, loading, error } as + | ErrorTranscript + | LoadingTranscript + | SuccessTranscript; }; export default useTranscript; diff --git a/www/app/[domain]/transcripts/useWebSockets.ts b/www/app/[domain]/transcripts/useWebSockets.ts index bcf6b163..cb74f86d 100644 --- a/www/app/[domain]/transcripts/useWebSockets.ts +++ b/www/app/[domain]/transcripts/useWebSockets.ts @@ -4,9 +4,10 @@ import { useError } from "../../(errors)/errorContext"; import { useRouter } from "next/navigation"; import { DomainContext } from "../domainContext"; -type UseWebSockets = { +export type UseWebSockets = { transcriptText: string; translateText: string; + title: string; topics: Topic[]; finalSummary: FinalSummary; status: Status; @@ -15,6 +16,7 @@ type UseWebSockets = { export const useWebSockets = (transcriptId: string | null): UseWebSockets => { const [transcriptText, setTranscriptText] = useState(""); const [translateText, setTranslateText] = useState(""); + const [title, setTitle] = useState(""); const [textQueue, setTextQueue] = useState([]); const [translationQueue, setTranslationQueue] = useState([]); const [isProcessing, setIsProcessing] = useState(false); @@ -24,7 +26,6 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { }); const [status, setStatus] = useState({ value: "initial" }); const { setError } = useError(); - const router = useRouter(); const { websocket_url } = useContext(DomainContext); @@ -294,7 +295,7 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { if (!transcriptId) return; const url = `${websocket_url}/v1/transcripts/${transcriptId}/events`; - const ws = new WebSocket(url); + let ws = new WebSocket(url); ws.onopen = () => { console.debug("WebSocket connection opened"); @@ -343,24 +344,23 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { case "FINAL_TITLE": console.debug("FINAL_TITLE event:", message.data); + if (message.data) { + setTitle(message.data.title); + } break; case "STATUS": console.log("STATUS event:", message.data); - if (message.data.value === "ended") { - const newUrl = "/transcripts/" + transcriptId; - router.push(newUrl); - console.debug("FINAL_LONG_SUMMARY event:", message.data); - } if (message.data.value === "error") { - const newUrl = "/transcripts/" + transcriptId; - router.push(newUrl); setError( Error("Websocket error status"), "There was an error processing this meeting.", ); } setStatus(message.data); + if (message.data.value === "ended") { + ws.close(); + } break; default: @@ -388,8 +388,16 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { default: setError( new Error(`WebSocket closed unexpectedly with code: ${event.code}`), + "Disconnected", ); } + console.log( + "Socket is closed. Reconnect will be attempted in 1 second.", + event.reason, + ); + setTimeout(function () { + ws = new WebSocket(url); + }, 1000); }; return () => { @@ -397,5 +405,5 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { }; }, [transcriptId]); - return { transcriptText, translateText, topics, finalSummary, status }; + return { transcriptText, translateText, topics, finalSummary, title, status }; }; From e98f1bf4bc7bed06fddecd22537ea819761b83b7 Mon Sep 17 00:00:00 2001 From: Sara Date: Mon, 13 Nov 2023 18:33:24 +0100 Subject: [PATCH 20/30] loading and redirecting front-end --- .../transcripts/[transcriptId]/page.tsx | 18 +++++-- .../[transcriptId]/record/page.tsx | 50 +++++++++++++------ www/app/[domain]/transcripts/finalSummary.tsx | 2 +- www/app/[domain]/transcripts/recorder.tsx | 13 +++-- www/app/[domain]/transcripts/useWebSockets.ts | 14 +++++- .../[domain]/transcripts/waveformLoading.tsx | 11 ++++ 6 files changed, 77 insertions(+), 31 deletions(-) create mode 100644 www/app/[domain]/transcripts/waveformLoading.tsx diff --git a/www/app/[domain]/transcripts/[transcriptId]/page.tsx b/www/app/[domain]/transcripts/[transcriptId]/page.tsx index add5a824..079b41e8 100644 --- a/www/app/[domain]/transcripts/[transcriptId]/page.tsx +++ b/www/app/[domain]/transcripts/[transcriptId]/page.tsx @@ -13,6 +13,7 @@ import ShareLink from "../shareLink"; import QRCode from "react-qr-code"; import TranscriptTitle from "../transcriptTitle"; import Player from "../player"; +import WaveformLoading from "../waveformLoading"; type TranscriptDetails = { params: { @@ -83,9 +84,9 @@ export default function TranscriptDetails(details: TranscriptDetails) { mediaDuration={transcript.response.duration} /> ) : mp3.error || waveform.error ? ( - "error loading this recording" +
"error loading this recording"
) : ( - "Loading Recording" + )}
@@ -104,10 +105,17 @@ export default function TranscriptDetails(details: TranscriptDetails) { summary={transcript.response.longSummary} transcriptId={transcript.response.id} /> - ) : transcript.response.status == "processing" ? ( - "Loading Transcript" ) : ( - "error final summary" +
+ {transcript.response.status == "processing" ? ( +

Loading Transcript

+ ) : ( +

+ There was an error generating the final summary, please + come back later +

+ )} +
)} diff --git a/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx b/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx index 10297f5c..36f4bbe9 100644 --- a/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx +++ b/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx @@ -11,9 +11,12 @@ import { Topic } from "../../webSocketTypes"; import LiveTrancription from "../../liveTranscription"; import DisconnectedIndicator from "../../disconnectedIndicator"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faGear } from "@fortawesome/free-solid-svg-icons"; +import { faGear, faSpinner } from "@fortawesome/free-solid-svg-icons"; import { lockWakeState, releaseWakeState } from "../../../../lib/wakeLock"; import { useRouter } from "next/navigation"; +import Player from "../../player"; +import useMp3 from "../../useMp3"; +import WaveformLoading from "../../waveformLoading"; type TranscriptDetails = { params: { @@ -42,8 +45,10 @@ const TranscriptRecord = (details: TranscriptDetails) => { const { audioDevices, getAudioStream } = useAudioDevice(); - const [hasRecorded, setHasRecorded] = useState(false); + const [recordedTime, setRecordedTime] = useState(0); + const [startTime, setStartTime] = useState(0); const [transcriptStarted, setTranscriptStarted] = useState(false); + const mp3 = useMp3(true, ""); const router = useRouter(); @@ -54,8 +59,6 @@ const TranscriptRecord = (details: TranscriptDetails) => { useEffect(() => { const statusToRedirect = ["ended", "error"]; - console.log(webSockets.status, "hey"); - console.log(transcript.response, "ho"); //TODO if has no topic and is error, get back to new if ( @@ -80,16 +83,31 @@ const TranscriptRecord = (details: TranscriptDetails) => { return ( <> - { - setStream(null); - setHasRecorded(true); - webRTC?.send(JSON.stringify({ cmd: "STOP" })); - }} - getAudioStream={getAudioStream} - audioDevices={audioDevices} - /> + {webSockets.waveform && mp3.media ? ( + + ) : recordedTime ? ( + + ) : ( + { + setStream(null); + setRecordedTime(Date.now() - startTime); + webRTC?.send(JSON.stringify({ cmd: "STOP" })); + }} + onRecord={() => { + setStartTime(Date.now()); + }} + getAudioStream={getAudioStream} + audioDevices={audioDevices} + /> + )}
{
- {!hasRecorded ? ( + {!recordedTime ? ( <> {transcriptStarted && (

Transcription

@@ -135,7 +153,7 @@ const TranscriptRecord = (details: TranscriptDetails) => { couple of minutes. Please do not navigate away from the page during this time.

- {/* TODO If login required remove last sentence */} + {/* NTH If login required remove last sentence */}
)} diff --git a/www/app/[domain]/transcripts/finalSummary.tsx b/www/app/[domain]/transcripts/finalSummary.tsx index 463f6100..e0d0f1c9 100644 --- a/www/app/[domain]/transcripts/finalSummary.tsx +++ b/www/app/[domain]/transcripts/finalSummary.tsx @@ -87,7 +87,7 @@ export default function FinalSummary(props: FinalSummaryProps) {
diff --git a/www/app/[domain]/transcripts/recorder.tsx b/www/app/[domain]/transcripts/recorder.tsx index 5b4420c4..e7c016a7 100644 --- a/www/app/[domain]/transcripts/recorder.tsx +++ b/www/app/[domain]/transcripts/recorder.tsx @@ -14,11 +14,11 @@ import { waveSurferStyles } from "../../styles/recorder"; import { useError } from "../../(errors)/errorContext"; type RecorderProps = { - setStream?: React.Dispatch>; - onStop?: () => void; - getAudioStream?: (deviceId) => Promise; - audioDevices?: Option[]; - mediaDuration?: number | null; + setStream: React.Dispatch>; + onStop: () => void; + onRecord?: () => void; + getAudioStream: (deviceId) => Promise; + audioDevices: Option[]; }; export default function Recorder(props: RecorderProps) { @@ -94,7 +94,6 @@ export default function Recorder(props: RecorderProps) { autoCenter: true, barWidth: 2, height: "auto", - duration: props.mediaDuration || 1, ...waveSurferStyles.player, }); @@ -161,10 +160,10 @@ export default function Recorder(props: RecorderProps) { setScreenMediaStream(null); setDestinationStream(null); } else { + if (props.onRecord) props.onRecord(); const stream = await getCurrentStream(); if (props.setStream) props.setStream(stream); - waveRegions?.clearRegions(); if (stream) { await record.startRecording(stream); setIsRecording(true); diff --git a/www/app/[domain]/transcripts/useWebSockets.ts b/www/app/[domain]/transcripts/useWebSockets.ts index cb74f86d..3f3d20fc 100644 --- a/www/app/[domain]/transcripts/useWebSockets.ts +++ b/www/app/[domain]/transcripts/useWebSockets.ts @@ -1,8 +1,8 @@ import { useContext, useEffect, useState } from "react"; import { Topic, FinalSummary, Status } from "./webSocketTypes"; import { useError } from "../../(errors)/errorContext"; -import { useRouter } from "next/navigation"; import { DomainContext } from "../domainContext"; +import { AudioWaveform } from "../../api"; export type UseWebSockets = { transcriptText: string; @@ -11,6 +11,7 @@ export type UseWebSockets = { topics: Topic[]; finalSummary: FinalSummary; status: Status; + waveform: AudioWaveform | null; }; export const useWebSockets = (transcriptId: string | null): UseWebSockets => { @@ -21,6 +22,7 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { const [translationQueue, setTranslationQueue] = useState([]); const [isProcessing, setIsProcessing] = useState(false); const [topics, setTopics] = useState([]); + const [waveform, setWaveForm] = useState(null); const [finalSummary, setFinalSummary] = useState({ summary: "", }); @@ -405,5 +407,13 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { }; }, [transcriptId]); - return { transcriptText, translateText, topics, finalSummary, title, status }; + return { + transcriptText, + translateText, + topics, + finalSummary, + title, + status, + waveform, + }; }; diff --git a/www/app/[domain]/transcripts/waveformLoading.tsx b/www/app/[domain]/transcripts/waveformLoading.tsx new file mode 100644 index 00000000..68e0c80f --- /dev/null +++ b/www/app/[domain]/transcripts/waveformLoading.tsx @@ -0,0 +1,11 @@ +import { faSpinner } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +export default () => ( +
+ +
+); From 4dbcb80228c14a4cce1c02ea23f0fab98537d571 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Wed, 15 Nov 2023 15:17:21 +0100 Subject: [PATCH 21/30] server: remove reference to banana.dev --- server/env.example | 18 ---- server/reflector/llm/llm_banana.py | 54 ------------ .../processors/audio_transcript_banana.py | 86 ------------------- server/reflector/settings.py | 17 ++-- 4 files changed, 6 insertions(+), 169 deletions(-) delete mode 100644 server/reflector/llm/llm_banana.py delete mode 100644 server/reflector/processors/audio_transcript_banana.py diff --git a/server/env.example b/server/env.example index 8c4dcdab..c5a38bf5 100644 --- a/server/env.example +++ b/server/env.example @@ -51,17 +51,6 @@ #TRANSLATE_URL=https://xxxxx--reflector-translator-web.modal.run #TRANSCRIPT_MODAL_API_KEY=xxxxx -## Using serverless banana.dev (require reflector-gpu-banana deployed) -## XXX this service is buggy do not use at the moment -## XXX it also require the audio to be saved to S3 -#TRANSCRIPT_BACKEND=banana -#TRANSCRIPT_URL=https://reflector-gpu-banana-xxxxx.run.banana.dev -#TRANSCRIPT_BANANA_API_KEY=xxx -#TRANSCRIPT_BANANA_MODEL_KEY=xxx -#TRANSCRIPT_STORAGE_AWS_ACCESS_KEY_ID=xxx -#TRANSCRIPT_STORAGE_AWS_SECRET_ACCESS_KEY=xxx -#TRANSCRIPT_STORAGE_AWS_BUCKET_NAME="reflector-bucket/chunks" - ## ======================================================= ## LLM backend ## @@ -78,13 +67,6 @@ #LLM_URL=https://xxxxxx--reflector-llm-web.modal.run #LLM_MODAL_API_KEY=xxx -## Using serverless banana.dev (require reflector-gpu-banana deployed) -## XXX this service is buggy do not use at the moment -#LLM_BACKEND=banana -#LLM_URL=https://reflector-gpu-banana-xxxxx.run.banana.dev -#LLM_BANANA_API_KEY=xxxxx -#LLM_BANANA_MODEL_KEY=xxxxx - ## Using OpenAI #LLM_BACKEND=openai #LLM_OPENAI_KEY=xxx diff --git a/server/reflector/llm/llm_banana.py b/server/reflector/llm/llm_banana.py deleted file mode 100644 index e0384770..00000000 --- a/server/reflector/llm/llm_banana.py +++ /dev/null @@ -1,54 +0,0 @@ -import httpx - -from reflector.llm.base import LLM -from reflector.settings import settings -from reflector.utils.retry import retry - - -class BananaLLM(LLM): - def __init__(self): - super().__init__() - self.timeout = settings.LLM_TIMEOUT - self.headers = { - "X-Banana-API-Key": settings.LLM_BANANA_API_KEY, - "X-Banana-Model-Key": settings.LLM_BANANA_MODEL_KEY, - } - - async def _generate( - self, prompt: str, gen_schema: dict | None, gen_cfg: dict | None, **kwargs - ): - json_payload = {"prompt": prompt} - if gen_schema: - json_payload["gen_schema"] = gen_schema - if gen_cfg: - json_payload["gen_cfg"] = gen_cfg - async with httpx.AsyncClient() as client: - response = await retry(client.post)( - settings.LLM_URL, - headers=self.headers, - json=json_payload, - timeout=self.timeout, - retry_timeout=300, # as per their sdk - ) - response.raise_for_status() - text = response.json()["text"] - return text - - -LLM.register("banana", BananaLLM) - -if __name__ == "__main__": - from reflector.logger import logger - - async def main(): - llm = BananaLLM() - prompt = llm.create_prompt( - instruct="Complete the following task", - text="Tell me a joke about programming.", - ) - result = await llm.generate(prompt=prompt, logger=logger) - print(result) - - import asyncio - - asyncio.run(main()) diff --git a/server/reflector/processors/audio_transcript_banana.py b/server/reflector/processors/audio_transcript_banana.py deleted file mode 100644 index fe339eea..00000000 --- a/server/reflector/processors/audio_transcript_banana.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Implementation using the GPU service from banana. - -API will be a POST request to TRANSCRIPT_URL: - -```json -{ - "audio_url": "https://...", - "audio_ext": "wav", - "timestamp": 123.456 - "language": "en" -} -``` - -""" - -from pathlib import Path - -import httpx -from reflector.processors.audio_transcript import AudioTranscriptProcessor -from reflector.processors.audio_transcript_auto import AudioTranscriptAutoProcessor -from reflector.processors.types import AudioFile, Transcript, Word -from reflector.settings import settings -from reflector.storage import Storage -from reflector.utils.retry import retry - - -class AudioTranscriptBananaProcessor(AudioTranscriptProcessor): - def __init__(self, banana_api_key: str, banana_model_key: str): - super().__init__() - self.transcript_url = settings.TRANSCRIPT_URL - self.timeout = settings.TRANSCRIPT_TIMEOUT - self.storage = Storage.get_instance( - settings.TRANSCRIPT_STORAGE_BACKEND, "TRANSCRIPT_STORAGE_" - ) - self.headers = { - "X-Banana-API-Key": banana_api_key, - "X-Banana-Model-Key": banana_model_key, - } - - async def _transcript(self, data: AudioFile): - async with httpx.AsyncClient() as client: - print(f"Uploading audio {data.path.name} to S3") - url = await self._upload_file(data.path) - - print(f"Try to transcribe audio {data.path.name}") - request_data = { - "audio_url": url, - "audio_ext": data.path.suffix[1:], - "timestamp": float(round(data.timestamp, 2)), - } - response = await retry(client.post)( - self.transcript_url, - json=request_data, - headers=self.headers, - timeout=self.timeout, - ) - - print(f"Transcript response: {response.status_code} {response.content}") - response.raise_for_status() - result = response.json() - transcript = Transcript( - text=result["text"], - words=[ - Word(text=word["text"], start=word["start"], end=word["end"]) - for word in result["words"] - ], - ) - - # remove audio file from S3 - await self._delete_file(data.path) - - return transcript - - @retry - async def _upload_file(self, path: Path) -> str: - upload_result = await self.storage.put_file(path.name, open(path, "rb")) - return upload_result.url - - @retry - async def _delete_file(self, path: Path): - await self.storage.delete_file(path.name) - return True - - -AudioTranscriptAutoProcessor.register("banana", AudioTranscriptBananaProcessor) diff --git a/server/reflector/settings.py b/server/reflector/settings.py index 013ab9b7..f9b997cb 100644 --- a/server/reflector/settings.py +++ b/server/reflector/settings.py @@ -41,7 +41,7 @@ class Settings(BaseSettings): AUDIO_BUFFER_SIZE: int = 256 * 960 # Audio Transcription - # backends: whisper, banana, modal + # backends: whisper, modal TRANSCRIPT_BACKEND: str = "whisper" TRANSCRIPT_URL: str | None = None TRANSCRIPT_TIMEOUT: int = 90 @@ -50,10 +50,6 @@ class Settings(BaseSettings): TRANSLATE_URL: str | None = None TRANSLATE_TIMEOUT: int = 90 - # Audio transcription banana.dev configuration - TRANSCRIPT_BANANA_API_KEY: str | None = None - TRANSCRIPT_BANANA_MODEL_KEY: str | None = None - # Audio transcription modal.com configuration TRANSCRIPT_MODAL_API_KEY: str | None = None @@ -61,13 +57,16 @@ class Settings(BaseSettings): TRANSCRIPT_STORAGE_BACKEND: str = "aws" # Storage configuration for AWS - TRANSCRIPT_STORAGE_AWS_BUCKET_NAME: str = "reflector-bucket/chunks" + TRANSCRIPT_STORAGE_AWS_BUCKET_NAME: str = "reflector-bucket" TRANSCRIPT_STORAGE_AWS_REGION: str = "us-east-1" TRANSCRIPT_STORAGE_AWS_ACCESS_KEY_ID: str | None = None TRANSCRIPT_STORAGE_AWS_SECRET_ACCESS_KEY: str | None = None + # Transcript MP3 storage + TRANSCRIPT_MP3_STORAGE_BACKEND: str = "aws" + # LLM - # available backend: openai, banana, modal, oobabooga + # available backend: openai, modal, oobabooga LLM_BACKEND: str = "oobabooga" # LLM common configuration @@ -82,10 +81,6 @@ class Settings(BaseSettings): LLM_TEMPERATURE: float = 0.7 ZEPHYR_LLM_URL: str | None = None - # LLM Banana configuration - LLM_BANANA_API_KEY: str | None = None - LLM_BANANA_MODEL_KEY: str | None = None - # LLM Modal configuration LLM_MODAL_API_KEY: str | None = None From e6c358a1d5ebed821fcae82763b85703bd54e6e5 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Wed, 15 Nov 2023 15:19:53 +0100 Subject: [PATCH 22/30] server: ability to deactivate diarization on the server --- server/reflector/pipelines/main_live_pipeline.py | 15 ++++++++------- server/reflector/settings.py | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/server/reflector/pipelines/main_live_pipeline.py b/server/reflector/pipelines/main_live_pipeline.py index 8c78c48f..316ecbcc 100644 --- a/server/reflector/pipelines/main_live_pipeline.py +++ b/server/reflector/pipelines/main_live_pipeline.py @@ -263,11 +263,7 @@ class PipelineMainLive(PipelineMainBase): TranscriptLinerProcessor(), TranscriptTranslatorProcessor.as_threaded(callback=self.on_transcript), TranscriptTopicDetectorProcessor.as_threaded(callback=self.on_topic), - BroadcastProcessor( - processors=[ - TranscriptFinalTitleProcessor.as_threaded(callback=self.on_title), - ] - ), + TranscriptFinalTitleProcessor.as_threaded(callback=self.on_title), ] pipeline = Pipeline(*processors) pipeline.options = self @@ -298,8 +294,13 @@ class PipelineMainDiarization(PipelineMainBase): # create a context for the whole rtc transaction # add a customised logger to the context self.prepare() - processors = [ - AudioDiarizationAutoProcessor(callback=self.on_topic), + processors = [] + if settings.DIARIZATION_ENABLED: + processors += [ + AudioDiarizationAutoProcessor(callback=self.on_topic), + ] + + processors += [ BroadcastProcessor( processors=[ TranscriptFinalLongSummaryProcessor.as_threaded( diff --git a/server/reflector/settings.py b/server/reflector/settings.py index f9b997cb..65412310 100644 --- a/server/reflector/settings.py +++ b/server/reflector/settings.py @@ -85,6 +85,7 @@ class Settings(BaseSettings): LLM_MODAL_API_KEY: str | None = None # Diarization + DIARIZATION_ENABLED: bool = True DIARIZATION_BACKEND: str = "modal" DIARIZATION_URL: str | None = None From 1fc261a66999df014820f07836d6300977977fe4 Mon Sep 17 00:00:00 2001 From: Sara Date: Wed, 15 Nov 2023 20:30:00 +0100 Subject: [PATCH 23/30] try to move waveform to pipeline --- server/reflector/db/transcripts.py | 25 +++++--------- .../reflector/pipelines/main_live_pipeline.py | 26 +++++++++++++-- .../processors/audio_waveform_processor.py | 33 +++++++++++++++++++ server/reflector/views/transcripts.py | 3 +- server/tests/test_transcripts_rtc_ws.py | 4 +++ www/app/lib/edgeConfig.ts | 4 +-- 6 files changed, 72 insertions(+), 23 deletions(-) create mode 100644 server/reflector/processors/audio_waveform_processor.py diff --git a/server/reflector/db/transcripts.py b/server/reflector/db/transcripts.py index 6ac2e32a..f0dbc277 100644 --- a/server/reflector/db/transcripts.py +++ b/server/reflector/db/transcripts.py @@ -10,7 +10,6 @@ from pydantic import BaseModel, Field from reflector.db import database, metadata from reflector.processors.types import Word as ProcessorWord from reflector.settings import settings -from reflector.utils.audio_waveform import get_audio_waveform transcripts = sqlalchemy.Table( "transcript", @@ -79,6 +78,14 @@ class TranscriptFinalTitle(BaseModel): title: str +class TranscriptDuration(BaseModel): + duration: float + + +class TranscriptWaveform(BaseModel): + waveform: list[float] + + class TranscriptEvent(BaseModel): event: str data: dict @@ -118,22 +125,6 @@ class Transcript(BaseModel): def topics_dump(self, mode="json"): return [topic.model_dump(mode=mode) for topic in self.topics] - def convert_audio_to_waveform(self, segments_count=256): - fn = self.audio_waveform_filename - if fn.exists(): - return - waveform = get_audio_waveform( - path=self.audio_mp3_filename, segments_count=segments_count - ) - try: - with open(fn, "w") as fd: - json.dump(waveform, fd) - except Exception: - # remove file if anything happen during the write - fn.unlink(missing_ok=True) - raise - return waveform - def unlink(self): self.data_path.unlink(missing_ok=True) diff --git a/server/reflector/pipelines/main_live_pipeline.py b/server/reflector/pipelines/main_live_pipeline.py index 8c78c48f..b0576b92 100644 --- a/server/reflector/pipelines/main_live_pipeline.py +++ b/server/reflector/pipelines/main_live_pipeline.py @@ -21,11 +21,13 @@ from pydantic import BaseModel from reflector.app import app from reflector.db.transcripts import ( Transcript, + TranscriptDuration, TranscriptFinalLongSummary, TranscriptFinalShortSummary, TranscriptFinalTitle, TranscriptText, TranscriptTopic, + TranscriptWaveform, transcripts_controller, ) from reflector.logger import logger @@ -45,6 +47,7 @@ from reflector.processors import ( TranscriptTopicDetectorProcessor, TranscriptTranslatorProcessor, ) +from reflector.processors.audio_waveform_processor import AudioWaveformProcessor from reflector.processors.types import AudioDiarizationInput from reflector.processors.types import ( TitleSummaryWithId as TitleSummaryWithIdProcessorType, @@ -230,15 +233,29 @@ class PipelineMainBase(PipelineRunner): data=final_short_summary, ) - async def on_duration(self, duration: float): + async def on_duration(self, data): async with self.transaction(): + duration = TranscriptDuration(duration=data) + transcript = await self.get_transcript() await transcripts_controller.update( transcript, { - "duration": duration, + "duration": duration.duration, }, ) + return await transcripts_controller.append_event( + transcript=transcript, event="DURATION", data=duration + ) + + async def on_waveform(self, data): + waveform = TranscriptWaveform(waveform=data) + + transcript = await self.get_transcript() + + return await transcripts_controller.append_event( + transcript=transcript, event="WAVEFORM", data=waveform + ) class PipelineMainLive(PipelineMainBase): @@ -266,6 +283,11 @@ class PipelineMainLive(PipelineMainBase): BroadcastProcessor( processors=[ TranscriptFinalTitleProcessor.as_threaded(callback=self.on_title), + AudioWaveformProcessor( + audio_path=transcript.audio_mp3_filename, + waveform_path=transcript.audio_waveform_filename, + on_waveform=self.on_waveform, + ), ] ), ] diff --git a/server/reflector/processors/audio_waveform_processor.py b/server/reflector/processors/audio_waveform_processor.py new file mode 100644 index 00000000..acce904a --- /dev/null +++ b/server/reflector/processors/audio_waveform_processor.py @@ -0,0 +1,33 @@ +import json +from pathlib import Path + +from reflector.processors.base import Processor +from reflector.processors.types import TitleSummary +from reflector.utils.audio_waveform import get_audio_waveform + + +class AudioWaveformProcessor(Processor): + """ + Write the waveform for the final audio + """ + + INPUT_TYPE = TitleSummary + + def __init__(self, audio_path: Path | str, waveform_path: str, **kwargs): + super().__init__(**kwargs) + if isinstance(audio_path, str): + audio_path = Path(audio_path) + if audio_path.suffix not in (".mp3", ".wav"): + raise ValueError("Only mp3 and wav files are supported") + self.audio_path = audio_path + self.waveform_path = waveform_path + + async def _push(self, _data): + self.waveform_path.parent.mkdir(parents=True, exist_ok=True) + self.logger.info("Waveform Processing Started") + waveform = get_audio_waveform(path=self.audio_path, segments_count=255) + + with open(self.waveform_path, "w") as fd: + json.dump(waveform, fd) + self.logger.info("Waveform Processing Finished") + await self.emit(waveform, name="waveform") diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 5de9ced3..77e4b0b7 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -22,7 +22,6 @@ from reflector.db.transcripts import ( from reflector.processors.types import Transcript as ProcessorTranscript from reflector.settings import settings from reflector.ws_manager import get_ws_manager -from starlette.concurrency import run_in_threadpool from ._range_requests_response import range_requests_response from .rtc_offer import RtcOffer, rtc_offer_base @@ -261,7 +260,7 @@ async def transcript_get_audio_waveform( if not transcript.audio_mp3_filename.exists(): raise HTTPException(status_code=404, detail="Audio not found") - await run_in_threadpool(transcript.convert_audio_to_waveform) + # await run_in_threadpool(transcript.convert_audio_to_waveform) return transcript.audio_waveform diff --git a/server/tests/test_transcripts_rtc_ws.py b/server/tests/test_transcripts_rtc_ws.py index cf2ea304..65660a5e 100644 --- a/server/tests/test_transcripts_rtc_ws.py +++ b/server/tests/test_transcripts_rtc_ws.py @@ -182,6 +182,10 @@ async def test_transcript_rtc_and_websocket( ev = events[eventnames.index("FINAL_TITLE")] assert ev["data"]["title"] == "LLM TITLE" + assert "WAVEFORM" in eventnames + ev = events[eventnames.index("FINAL_TITLE")] + assert ev["data"]["title"] == "LLM TITLE" + # check status order statuses = [e["data"]["value"] for e in events if e["event"] == "STATUS"] assert statuses.index("recording") < statuses.index("processing") diff --git a/www/app/lib/edgeConfig.ts b/www/app/lib/edgeConfig.ts index 5527121a..1140e555 100644 --- a/www/app/lib/edgeConfig.ts +++ b/www/app/lib/edgeConfig.ts @@ -3,9 +3,9 @@ import { isDevelopment } from "./utils"; const localConfig = { features: { - requireLogin: true, + requireLogin: false, privacy: true, - browse: true, + browse: false, }, api_url: "http://127.0.0.1:1250", websocket_url: "ws://127.0.0.1:1250", From 8b8e92ceac7a121bd16909ea2d8401a889ae44cb Mon Sep 17 00:00:00 2001 From: Sara Date: Wed, 15 Nov 2023 20:34:45 +0100 Subject: [PATCH 24/30] replace history instead of pushing --- www/app/[domain]/transcripts/[transcriptId]/record/page.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx b/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx index 36f4bbe9..147f8827 100644 --- a/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx +++ b/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx @@ -69,9 +69,9 @@ const TranscriptRecord = (details: TranscriptDetails) => { // Shallow redirection does not work on NextJS 13 // https://github.com/vercel/next.js/discussions/48110 // https://github.com/vercel/next.js/discussions/49540 - router.push(newUrl, undefined); + router.replace(newUrl); // history.replaceState({}, "", newUrl); - } + } // history.replaceState({}, "", newUrl); }, [webSockets.status.value, transcript.response?.status]); useEffect(() => { From 2e1b4c2c6895229138ed46e0e3c082d3d13a0535 Mon Sep 17 00:00:00 2001 From: Andreas Bonini <78463782+AndreasBonini@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:19:24 +0700 Subject: [PATCH 25/30] Update deploy.yml --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ecac11b4..1ab6a031 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,7 +3,7 @@ name: Deploy to Amazon ECS on: [workflow_dispatch] env: - # 384658522150.dkr.ecr.us-east-1.amazonaws.com/reflector + # 950402358378.dkr.ecr.us-east-1.amazonaws.com/reflector AWS_REGION: us-east-1 ECR_REPOSITORY: reflector From 317f649b9c62fc13faf77bd483a13524fb84371e Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 16 Nov 2023 19:22:55 +0700 Subject: [PATCH 26/30] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a43e88f7..c3b01d5a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ server/.env .env server/exportdanswer +.vercel +.env*.local From 99796d045b2984b6eeaf36e3c9aa82336a9a936b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:32:00 +0000 Subject: [PATCH 27/30] build(deps): bump @sentry/nextjs from 7.64.0 to 7.77.0 in /www Bumps [@sentry/nextjs](https://github.com/getsentry/sentry-javascript) from 7.64.0 to 7.77.0. - [Release notes](https://github.com/getsentry/sentry-javascript/releases) - [Changelog](https://github.com/getsentry/sentry-javascript/blob/develop/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-javascript/compare/7.64.0...7.77.0) --- updated-dependencies: - dependency-name: "@sentry/nextjs" dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- www/package.json | 2 +- www/yarn.lock | 205 ++++++++++++++++++++++++----------------------- 2 files changed, 106 insertions(+), 101 deletions(-) diff --git a/www/package.json b/www/package.json index 55c7df73..7e43fd8c 100644 --- a/www/package.json +++ b/www/package.json @@ -15,7 +15,7 @@ "@fortawesome/fontawesome-svg-core": "^6.4.0", "@fortawesome/free-solid-svg-icons": "^6.4.0", "@fortawesome/react-fontawesome": "^0.2.0", - "@sentry/nextjs": "^7.64.0", + "@sentry/nextjs": "^7.77.0", "@vercel/edge-config": "^0.4.1", "autoprefixer": "10.4.14", "axios": "^1.4.0", diff --git a/www/yarn.lock b/www/yarn.lock index 8ec03382..21d6ba4c 100644 --- a/www/yarn.lock +++ b/www/yarn.lock @@ -262,27 +262,25 @@ estree-walker "^2.0.2" picomatch "^2.3.1" -"@sentry-internal/tracing@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.64.0.tgz#3e110473b8edf805b799cc91d6ee592830237bb4" - integrity sha512-1XE8W6ki7hHyBvX9hfirnGkKDBKNq3bDJyXS86E0bYVDl94nvbRM9BD9DHsCFetqYkVm1yDGEK+6aUVs4CztoQ== +"@sentry-internal/tracing@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.77.0.tgz#f3d82486f8934a955b3dd2aa54c8d29586e42a37" + integrity sha512-8HRF1rdqWwtINqGEdx8Iqs9UOP/n8E0vXUu3Nmbqj4p5sQPA7vvCfq+4Y4rTqZFc7sNdFpDsRION5iQEh8zfZw== dependencies: - "@sentry/core" "7.64.0" - "@sentry/types" "7.64.0" - "@sentry/utils" "7.64.0" - tslib "^2.4.1 || ^1.9.3" + "@sentry/core" "7.77.0" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" -"@sentry/browser@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/browser/-/browser-7.64.0.tgz#76db08a5d32ffe7c5aa907f258e6c845ce7f10d7" - integrity sha512-lB2IWUkZavEDclxfLBp554dY10ZNIEvlDZUWWathW+Ws2wRb6PNLtuPUNu12R7Q7z0xpkOLrM1kRNN0OdldgKA== +"@sentry/browser@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/browser/-/browser-7.77.0.tgz#155440f1a0d3a1bbd5d564c28d6b0c9853a51d72" + integrity sha512-nJ2KDZD90H8jcPx9BysQLiQW+w7k7kISCWeRjrEMJzjtge32dmHA8G4stlUTRIQugy5F+73cOayWShceFP7QJQ== dependencies: - "@sentry-internal/tracing" "7.64.0" - "@sentry/core" "7.64.0" - "@sentry/replay" "7.64.0" - "@sentry/types" "7.64.0" - "@sentry/utils" "7.64.0" - tslib "^2.4.1 || ^1.9.3" + "@sentry-internal/tracing" "7.77.0" + "@sentry/core" "7.77.0" + "@sentry/replay" "7.77.0" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" "@sentry/cli@^1.74.6": version "1.75.2" @@ -296,89 +294,94 @@ proxy-from-env "^1.1.0" which "^2.0.2" -"@sentry/core@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.64.0.tgz#9d61cdc29ba299dedbdcbe01cfadf94bd0b7df48" - integrity sha512-IzmEyl5sNG7NyEFiyFHEHC+sizsZp9MEw1+RJRLX6U5RITvcsEgcajSkHQFafaBPzRrcxZMdm47Cwhl212LXcw== +"@sentry/core@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.77.0.tgz#21100843132beeeff42296c8370cdcc7aa1d8510" + integrity sha512-Tj8oTYFZ/ZD+xW8IGIsU6gcFXD/gfE+FUxUaeSosd9KHwBQNOLhZSsYo/tTVf/rnQI/dQnsd4onPZLiL+27aTg== dependencies: - "@sentry/types" "7.64.0" - "@sentry/utils" "7.64.0" - tslib "^2.4.1 || ^1.9.3" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" -"@sentry/integrations@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/integrations/-/integrations-7.64.0.tgz#a392ddeebeec0c08ae5ca1f544c80ab15977fe10" - integrity sha512-6gbSGiruOifAmLtXw//Za19GWiL5qugDMEFxSvc5WrBWb+A8UK+foPn3K495OcivLS68AmqAQCUGb+6nlVowwA== +"@sentry/integrations@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/integrations/-/integrations-7.77.0.tgz#f2717e05cb7c69363316ccd34096b2ea07ae4c59" + integrity sha512-P055qXgBHeZNKnnVEs5eZYLdy6P49Zr77A1aWJuNih/EenzMy922GOeGy2mF6XYrn1YJSjEwsNMNsQkcvMTK8Q== dependencies: - "@sentry/types" "7.64.0" - "@sentry/utils" "7.64.0" + "@sentry/core" "7.77.0" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" localforage "^1.8.1" - tslib "^2.4.1 || ^1.9.3" -"@sentry/nextjs@^7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/nextjs/-/nextjs-7.64.0.tgz#5c0bd7ccc6637e0b925dec25ca247dcb8476663c" - integrity sha512-hKlIQpFugdRlWj0wcEG9I8JyVm/osdsE72zwMBGnmCw/jf7U63vjOjfxMe/gRuvllCf/AvoGHEkR5jPufcO+bw== +"@sentry/nextjs@^7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/nextjs/-/nextjs-7.77.0.tgz#036b1c45dd106e01d44967c97985464e108922be" + integrity sha512-8tYPBt5luFjrng1sAMJqNjM9sq80q0jbt6yariADU9hEr7Zk8YqFaOI2/Q6yn9dZ6XyytIRtLEo54kk2AO94xw== dependencies: "@rollup/plugin-commonjs" "24.0.0" - "@sentry/core" "7.64.0" - "@sentry/integrations" "7.64.0" - "@sentry/node" "7.64.0" - "@sentry/react" "7.64.0" - "@sentry/types" "7.64.0" - "@sentry/utils" "7.64.0" + "@sentry/core" "7.77.0" + "@sentry/integrations" "7.77.0" + "@sentry/node" "7.77.0" + "@sentry/react" "7.77.0" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" + "@sentry/vercel-edge" "7.77.0" "@sentry/webpack-plugin" "1.20.0" chalk "3.0.0" + resolve "1.22.8" rollup "2.78.0" stacktrace-parser "^0.1.10" - tslib "^2.4.1 || ^1.9.3" -"@sentry/node@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/node/-/node-7.64.0.tgz#c6f7a67c1442324298f0525e7191bc18572ee1ce" - integrity sha512-wRi0uTnp1WSa83X2yLD49tV9QPzGh5e42IKdIDBiQ7lV9JhLILlyb34BZY1pq6p4dp35yDasDrP3C7ubn7wo6A== +"@sentry/node@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/node/-/node-7.77.0.tgz#a247452779a5bcb55724457707286e3e4a29dbbe" + integrity sha512-Ob5tgaJOj0OYMwnocc6G/CDLWC7hXfVvKX/ofkF98+BbN/tQa5poL+OwgFn9BA8ud8xKzyGPxGU6LdZ8Oh3z/g== dependencies: - "@sentry-internal/tracing" "7.64.0" - "@sentry/core" "7.64.0" - "@sentry/types" "7.64.0" - "@sentry/utils" "7.64.0" - cookie "^0.4.1" + "@sentry-internal/tracing" "7.77.0" + "@sentry/core" "7.77.0" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" https-proxy-agent "^5.0.0" - lru_map "^0.3.3" - tslib "^2.4.1 || ^1.9.3" -"@sentry/react@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/react/-/react-7.64.0.tgz#edee24ac232990204e0fb43dd83994642d4b0f54" - integrity sha512-wOyJUQi7OoT1q+F/fVVv1fzbyO4OYbTu6m1DliLOGQPGEHPBsgPc722smPIExd1/rAMK/FxOuNN5oNhubH8nhg== +"@sentry/react@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/react/-/react-7.77.0.tgz#9da14e4b21eae4b5a6306d39bb7c42ef0827d2c2" + integrity sha512-Q+htKzib5em0MdaQZMmPomaswaU3xhcVqmLi2CxqQypSjbYgBPPd+DuhrXKoWYLDDkkbY2uyfe4Lp3yLRWeXYw== dependencies: - "@sentry/browser" "7.64.0" - "@sentry/types" "7.64.0" - "@sentry/utils" "7.64.0" + "@sentry/browser" "7.77.0" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" hoist-non-react-statics "^3.3.2" - tslib "^2.4.1 || ^1.9.3" -"@sentry/replay@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/replay/-/replay-7.64.0.tgz#bdf09b0c4712f9dc6b24b3ebefa55a4ac76708e6" - integrity sha512-alaMCZDZhaAVmEyiUnszZnvfdbiZx5MmtMTGrlDd7tYq3K5OA9prdLqqlmfIJYBfYtXF3lD0iZFphOZQD+4CIw== +"@sentry/replay@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/replay/-/replay-7.77.0.tgz#21d242c9cd70a7235237216174873fd140b6eb80" + integrity sha512-M9Ik2J5ekl+C1Och3wzLRZVaRGK33BlnBwfwf3qKjgLDwfKW+1YkwDfTHbc2b74RowkJbOVNcp4m8ptlehlSaQ== dependencies: - "@sentry/core" "7.64.0" - "@sentry/types" "7.64.0" - "@sentry/utils" "7.64.0" + "@sentry-internal/tracing" "7.77.0" + "@sentry/core" "7.77.0" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" -"@sentry/types@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.64.0.tgz#21fc545ea05c3c8c4c3e518583eca1a8c5429506" - integrity sha512-LqjQprWXjUFRmzIlUjyA+KL+38elgIYmAeoDrdyNVh8MK5IC1W2Lh1Q87b4yOiZeMiIhIVNBd7Ecoh2rodGrGA== +"@sentry/types@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.77.0.tgz#c5d00fe547b89ccde59cdea59143bf145cee3144" + integrity sha512-nfb00XRJVi0QpDHg+JkqrmEBHsqBnxJu191Ded+Cs1OJ5oPXEW6F59LVcBScGvMqe+WEk1a73eH8XezwfgrTsA== -"@sentry/utils@7.64.0": - version "7.64.0" - resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.64.0.tgz#6fe3ce9a56d3433ed32119f914907361a54cc184" - integrity sha512-HRlM1INzK66Gt+F4vCItiwGKAng4gqzCR4C5marsL3qv6SrKH98dQnCGYgXluSWaaa56h97FRQu7TxCk6jkSvQ== +"@sentry/utils@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.77.0.tgz#1f88501f0b8777de31b371cf859d13c82ebe1379" + integrity sha512-NmM2kDOqVchrey3N5WSzdQoCsyDkQkiRxExPaNI2oKQ/jMWHs9yt0tSy7otPBcXs0AP59ihl75Bvm1tDRcsp5g== dependencies: - "@sentry/types" "7.64.0" - tslib "^2.4.1 || ^1.9.3" + "@sentry/types" "7.77.0" + +"@sentry/vercel-edge@7.77.0": + version "7.77.0" + resolved "https://registry.yarnpkg.com/@sentry/vercel-edge/-/vercel-edge-7.77.0.tgz#6a90a869878e4e78803c4331c30aea841fcc6a73" + integrity sha512-ffddPCgxVeAccPYuH5sooZeHBqDuJ9OIhIRYKoDi4TvmwAzWo58zzZWhRpkHqHgIQdQvhLVZ5F+FSQVWnYSOkw== + dependencies: + "@sentry/core" "7.77.0" + "@sentry/types" "7.77.0" + "@sentry/utils" "7.77.0" "@sentry/webpack-plugin@1.20.0": version "1.20.0" @@ -860,11 +863,6 @@ console.table@0.10.0: dependencies: easy-table "1.1.0" -cookie@^0.4.1: - version "0.4.2" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" - integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== - cookiejar@^2.1.4: version "2.1.4" resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.4.tgz#ee669c1fea2cf42dc31585469d193fef0d65771b" @@ -1096,6 +1094,11 @@ function-bind@^1.1.1: resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + get-browser-rtc@^1.1.0: version "1.1.0" resolved "https://registry.npmjs.org/get-browser-rtc/-/get-browser-rtc-1.1.0.tgz" @@ -1185,6 +1188,13 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +hasown@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.0.tgz#f4c513d454a57b7c7e1650778de226b11700546c" + integrity sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA== + dependencies: + function-bind "^1.1.2" + hast-util-to-jsx-runtime@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.2.0.tgz#ffd59bfcf0eb8321c6ed511bfc4b399ac3404bc2" @@ -1307,12 +1317,12 @@ is-binary-path@~2.1.0: dependencies: binary-extensions "^2.0.0" -is-core-module@^2.11.0: - version "2.12.1" - resolved "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.1.tgz" - integrity sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg== +is-core-module@^2.13.0: + version "2.13.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384" + integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw== dependencies: - has "^1.0.3" + hasown "^2.0.0" is-extglob@^2.1.1: version "2.1.1" @@ -1465,11 +1475,6 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -lru_map@^0.3.3: - version "0.3.3" - resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd" - integrity sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ== - magic-string@^0.27.0: version "0.27.0" resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.27.0.tgz#e4a3413b4bab6d98d2becffd48b4a257effdbbf3" @@ -2171,12 +2176,12 @@ require-directory@^2.1.1: resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== -resolve@^1.1.7, resolve@^1.22.2: - version "1.22.2" - resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz" - integrity sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g== +resolve@1.22.8, resolve@^1.1.7, resolve@^1.22.2: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== dependencies: - is-core-module "^2.11.0" + is-core-module "^2.13.0" path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" @@ -2512,7 +2517,7 @@ tslib@^1.9.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== -tslib@^2.1.0, "tslib@^2.4.1 || ^1.9.3": +tslib@^2.1.0: version "2.6.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.1.tgz#fd8c9a0ff42590b25703c0acb3de3d3f4ede0410" integrity sha512-t0hLfiEKfMUoqhG+U1oid7Pva4bbDPHYfJNiB7BiIjRkj1pyC++4N3huJfqY6aRH6VTB0rvtzQwjM4K6qpfOig== From a846e38fbdeb77aad23282a8068b08e4cd6b3921 Mon Sep 17 00:00:00 2001 From: Sara Date: Fri, 17 Nov 2023 13:38:32 +0100 Subject: [PATCH 28/30] fix waveform in pipeline --- .../reflector/pipelines/main_live_pipeline.py | 15 +++++++++------ .../processors/audio_waveform_processor.py | 5 ++++- server/tests/test_transcripts_audio_download.py | 12 ------------ server/tests/test_transcripts_rtc_ws.py | 17 ++++++++++++----- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/server/reflector/pipelines/main_live_pipeline.py b/server/reflector/pipelines/main_live_pipeline.py index b0576b92..fece6da5 100644 --- a/server/reflector/pipelines/main_live_pipeline.py +++ b/server/reflector/pipelines/main_live_pipeline.py @@ -233,6 +233,7 @@ class PipelineMainBase(PipelineRunner): data=final_short_summary, ) + @broadcast_to_sockets async def on_duration(self, data): async with self.transaction(): duration = TranscriptDuration(duration=data) @@ -248,14 +249,16 @@ class PipelineMainBase(PipelineRunner): transcript=transcript, event="DURATION", data=duration ) + @broadcast_to_sockets async def on_waveform(self, data): - waveform = TranscriptWaveform(waveform=data) + async with self.transaction(): + waveform = TranscriptWaveform(waveform=data) - transcript = await self.get_transcript() + transcript = await self.get_transcript() - return await transcripts_controller.append_event( - transcript=transcript, event="WAVEFORM", data=waveform - ) + return await transcripts_controller.append_event( + transcript=transcript, event="WAVEFORM", data=waveform + ) class PipelineMainLive(PipelineMainBase): @@ -283,7 +286,7 @@ class PipelineMainLive(PipelineMainBase): BroadcastProcessor( processors=[ TranscriptFinalTitleProcessor.as_threaded(callback=self.on_title), - AudioWaveformProcessor( + AudioWaveformProcessor.as_threaded( audio_path=transcript.audio_mp3_filename, waveform_path=transcript.audio_waveform_filename, on_waveform=self.on_waveform, diff --git a/server/reflector/processors/audio_waveform_processor.py b/server/reflector/processors/audio_waveform_processor.py index acce904a..f1a24ffd 100644 --- a/server/reflector/processors/audio_waveform_processor.py +++ b/server/reflector/processors/audio_waveform_processor.py @@ -22,7 +22,7 @@ class AudioWaveformProcessor(Processor): self.audio_path = audio_path self.waveform_path = waveform_path - async def _push(self, _data): + async def _flush(self): self.waveform_path.parent.mkdir(parents=True, exist_ok=True) self.logger.info("Waveform Processing Started") waveform = get_audio_waveform(path=self.audio_path, segments_count=255) @@ -31,3 +31,6 @@ class AudioWaveformProcessor(Processor): json.dump(waveform, fd) self.logger.info("Waveform Processing Finished") await self.emit(waveform, name="waveform") + + async def _push(_self, _data): + return diff --git a/server/tests/test_transcripts_audio_download.py b/server/tests/test_transcripts_audio_download.py index 69ae5f65..28f83fff 100644 --- a/server/tests/test_transcripts_audio_download.py +++ b/server/tests/test_transcripts_audio_download.py @@ -118,15 +118,3 @@ async def test_transcript_audio_download_range_with_seek( assert response.status_code == 206 assert response.headers["content-type"] == content_type assert response.headers["content-range"].startswith("bytes 100-") - - -@pytest.mark.asyncio -async def test_transcript_audio_download_waveform(fake_transcript): - from reflector.app import app - - ac = AsyncClient(app=app, base_url="http://test/v1") - response = await ac.get(f"/transcripts/{fake_transcript.id}/audio/waveform") - assert response.status_code == 200 - assert response.headers["content-type"] == "application/json" - assert isinstance(response.json()["data"], list) - assert len(response.json()["data"]) >= 255 diff --git a/server/tests/test_transcripts_rtc_ws.py b/server/tests/test_transcripts_rtc_ws.py index 65660a5e..b33b1db5 100644 --- a/server/tests/test_transcripts_rtc_ws.py +++ b/server/tests/test_transcripts_rtc_ws.py @@ -183,8 +183,14 @@ async def test_transcript_rtc_and_websocket( assert ev["data"]["title"] == "LLM TITLE" assert "WAVEFORM" in eventnames - ev = events[eventnames.index("FINAL_TITLE")] - assert ev["data"]["title"] == "LLM TITLE" + ev = events[eventnames.index("WAVEFORM")] + assert isinstance(ev["data"]["waveform"], list) + assert len(ev["data"]["waveform"]) >= 250 + waveform_resp = await ac.get(f"/transcripts/{tid}/audio/waveform") + assert waveform_resp.status_code == 200 + assert waveform_resp.headers["content-type"] == "application/json" + assert isinstance(waveform_resp.json()["data"], list) + assert len(waveform_resp.json()["data"]) >= 250 # check status order statuses = [e["data"]["value"] for e in events if e["event"] == "STATUS"] @@ -197,11 +203,12 @@ async def test_transcript_rtc_and_websocket( # check on the latest response that the audio duration is > 0 assert resp.json()["duration"] > 0 + assert "DURATION" in eventnames # check that audio/mp3 is available - resp = await ac.get(f"/transcripts/{tid}/audio/mp3") - assert resp.status_code == 200 - assert resp.headers["Content-Type"] == "audio/mpeg" + audio_resp = await ac.get(f"/transcripts/{tid}/audio/mp3") + assert audio_resp.status_code == 200 + assert audio_resp.headers["Content-Type"] == "audio/mpeg" @pytest.mark.usefixtures("celery_session_app") From a816e00769faba40098acd36085469a23089f614 Mon Sep 17 00:00:00 2001 From: Sara Date: Fri, 17 Nov 2023 15:16:53 +0100 Subject: [PATCH 29/30] get waveform from socket --- .../transcripts/[transcriptId]/page.tsx | 7 ++--- .../[transcriptId]/record/page.tsx | 16 ++++++---- www/app/[domain]/transcripts/player.tsx | 5 ++-- www/app/[domain]/transcripts/useMp3.ts | 29 ++++++++----------- www/app/[domain]/transcripts/useWebSockets.ts | 21 +++++++++++++- 5 files changed, 48 insertions(+), 30 deletions(-) diff --git a/www/app/[domain]/transcripts/[transcriptId]/page.tsx b/www/app/[domain]/transcripts/[transcriptId]/page.tsx index 079b41e8..b0986e94 100644 --- a/www/app/[domain]/transcripts/[transcriptId]/page.tsx +++ b/www/app/[domain]/transcripts/[transcriptId]/page.tsx @@ -30,7 +30,7 @@ export default function TranscriptDetails(details: TranscriptDetails) { const topics = useTopics(protectedPath, transcriptId); const waveform = useWaveform(protectedPath, transcriptId); const useActiveTopic = useState(null); - const mp3 = useMp3(protectedPath, transcriptId); + const mp3 = useMp3(transcriptId); if (transcript?.error || topics?.error) { return ( @@ -59,7 +59,6 @@ export default function TranscriptDetails(details: TranscriptDetails) { .join("\n\n") .replace(/ +/g, " ") .trim() || ""; - console.log("calf full transcript"); return ( <> @@ -79,11 +78,11 @@ export default function TranscriptDetails(details: TranscriptDetails) { - ) : mp3.error || waveform.error ? ( + ) : waveform.error ? (
"error loading this recording"
) : ( diff --git a/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx b/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx index 147f8827..2c5b73e0 100644 --- a/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx +++ b/www/app/[domain]/transcripts/[transcriptId]/record/page.tsx @@ -11,11 +11,11 @@ import { Topic } from "../../webSocketTypes"; import LiveTrancription from "../../liveTranscription"; import DisconnectedIndicator from "../../disconnectedIndicator"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faGear, faSpinner } from "@fortawesome/free-solid-svg-icons"; +import { faGear } from "@fortawesome/free-solid-svg-icons"; import { lockWakeState, releaseWakeState } from "../../../../lib/wakeLock"; import { useRouter } from "next/navigation"; import Player from "../../player"; -import useMp3 from "../../useMp3"; +import useMp3, { Mp3Response } from "../../useMp3"; import WaveformLoading from "../../waveformLoading"; type TranscriptDetails = { @@ -48,7 +48,7 @@ const TranscriptRecord = (details: TranscriptDetails) => { const [recordedTime, setRecordedTime] = useState(0); const [startTime, setStartTime] = useState(0); const [transcriptStarted, setTranscriptStarted] = useState(false); - const mp3 = useMp3(true, ""); + let mp3 = useMp3(details.params.transcriptId, true); const router = useRouter(); @@ -74,6 +74,12 @@ const TranscriptRecord = (details: TranscriptDetails) => { } // history.replaceState({}, "", newUrl); }, [webSockets.status.value, transcript.response?.status]); + useEffect(() => { + if (webSockets.duration) { + mp3.getNow(); + } + }, [webSockets.duration]); + useEffect(() => { lockWakeState(); return () => { @@ -83,13 +89,13 @@ const TranscriptRecord = (details: TranscriptDetails) => { return ( <> - {webSockets.waveform && mp3.media ? ( + {webSockets.waveform && webSockets.duration && mp3?.media ? ( ) : recordedTime ? ( diff --git a/www/app/[domain]/transcripts/player.tsx b/www/app/[domain]/transcripts/player.tsx index 6143836e..02151a68 100644 --- a/www/app/[domain]/transcripts/player.tsx +++ b/www/app/[domain]/transcripts/player.tsx @@ -14,7 +14,7 @@ type PlayerProps = { Topic | null, React.Dispatch>, ]; - waveform: AudioWaveform; + waveform: AudioWaveform["data"]; media: HTMLMediaElement; mediaDuration: number; }; @@ -29,7 +29,6 @@ export default function Player(props: PlayerProps) { ); const [activeTopic, setActiveTopic] = props.useActiveTopic; const topicsRef = useRef(props.topics); - // Waveform setup useEffect(() => { if (waveformRef.current) { @@ -40,7 +39,7 @@ export default function Player(props: PlayerProps) { // This is not ideal, but it works for now. const _wavesurfer = WaveSurfer.create({ container: waveformRef.current, - peaks: props.waveform.data, + peaks: props.waveform, hideScrollbar: true, autoCenter: true, barWidth: 2, diff --git a/www/app/[domain]/transcripts/useMp3.ts b/www/app/[domain]/transcripts/useMp3.ts index 570a6a25..23249f94 100644 --- a/www/app/[domain]/transcripts/useMp3.ts +++ b/www/app/[domain]/transcripts/useMp3.ts @@ -1,24 +1,19 @@ import { useContext, useEffect, useState } from "react"; -import { useError } from "../../(errors)/errorContext"; import { DomainContext } from "../domainContext"; import getApi from "../../lib/getApi"; import { useFiefAccessTokenInfo } from "@fief/fief/build/esm/nextjs/react"; -import { shouldShowError } from "../../lib/errorUtils"; -type Mp3Response = { - url: string | null; +export type Mp3Response = { media: HTMLMediaElement | null; loading: boolean; - error: Error | null; + getNow: () => void; }; -const useMp3 = (protectedPath: boolean, id: string): Mp3Response => { - const [url, setUrl] = useState(null); +const useMp3 = (id: string, waiting?: boolean): Mp3Response => { const [media, setMedia] = useState(null); + const [later, setLater] = useState(waiting); const [loading, setLoading] = useState(false); - const [error, setErrorState] = useState(null); - const { setError } = useError(); - const api = getApi(protectedPath); + const api = getApi(true); const { api_url } = useContext(DomainContext); const accessTokenInfo = useFiefAccessTokenInfo(); const [serviceWorkerReady, setServiceWorkerReady] = useState(false); @@ -42,8 +37,8 @@ const useMp3 = (protectedPath: boolean, id: string): Mp3Response => { }); }, [navigator.serviceWorker, serviceWorkerReady, accessTokenInfo]); - const getMp3 = (id: string) => { - if (!id || !api) return; + useEffect(() => { + if (!id || !api || later) return; // createa a audio element and set the source setLoading(true); @@ -53,13 +48,13 @@ const useMp3 = (protectedPath: boolean, id: string): Mp3Response => { audioElement.preload = "auto"; setMedia(audioElement); setLoading(false); + }, [id, api, later]); + + const getNow = () => { + setLater(false); }; - useEffect(() => { - getMp3(id); - }, [id, api]); - - return { url, media, loading, error }; + return { media, loading, getNow }; }; export default useMp3; diff --git a/www/app/[domain]/transcripts/useWebSockets.ts b/www/app/[domain]/transcripts/useWebSockets.ts index 3f3d20fc..f33a1347 100644 --- a/www/app/[domain]/transcripts/useWebSockets.ts +++ b/www/app/[domain]/transcripts/useWebSockets.ts @@ -11,7 +11,8 @@ export type UseWebSockets = { topics: Topic[]; finalSummary: FinalSummary; status: Status; - waveform: AudioWaveform | null; + waveform: AudioWaveform["data"] | null; + duration: number | null; }; export const useWebSockets = (transcriptId: string | null): UseWebSockets => { @@ -23,6 +24,7 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { const [isProcessing, setIsProcessing] = useState(false); const [topics, setTopics] = useState([]); const [waveform, setWaveForm] = useState(null); + const [duration, setDuration] = useState(null); const [finalSummary, setFinalSummary] = useState({ summary: "", }); @@ -351,6 +353,22 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { } break; + case "WAVEFORM": + console.debug( + "WAVEFORM event length:", + message.data.waveform.length, + ); + if (message.data) { + setWaveForm(message.data.waveform); + } + break; + case "DURATION": + console.debug("DURATION event:", message.data); + if (message.data) { + setDuration(message.data.duration); + } + break; + case "STATUS": console.log("STATUS event:", message.data); if (message.data.value === "error") { @@ -415,5 +433,6 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { title, status, waveform, + duration, }; }; From c40e0970b7d9dfe4d3df2cb159b7ffbecfece077 Mon Sep 17 00:00:00 2001 From: Sara Date: Mon, 20 Nov 2023 20:13:18 +0100 Subject: [PATCH 30/30] review fixes --- server/reflector/views/transcripts.py | 2 -- www/app/[domain]/transcripts/useWebSockets.ts | 17 +++++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 77e4b0b7..6909b8ae 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -260,8 +260,6 @@ async def transcript_get_audio_waveform( if not transcript.audio_mp3_filename.exists(): raise HTTPException(status_code=404, detail="Audio not found") - # await run_in_threadpool(transcript.convert_audio_to_waveform) - return transcript.audio_waveform diff --git a/www/app/[domain]/transcripts/useWebSockets.ts b/www/app/[domain]/transcripts/useWebSockets.ts index f33a1347..f289adbb 100644 --- a/www/app/[domain]/transcripts/useWebSockets.ts +++ b/www/app/[domain]/transcripts/useWebSockets.ts @@ -402,22 +402,19 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { console.debug("WebSocket connection closed"); switch (event.code) { case 1000: // Normal Closure: - case 1001: // Going Away: - case 1005: - break; default: setError( new Error(`WebSocket closed unexpectedly with code: ${event.code}`), "Disconnected", ); + console.log( + "Socket is closed. Reconnect will be attempted in 1 second.", + event.reason, + ); + setTimeout(function () { + ws = new WebSocket(url); + }, 1000); } - console.log( - "Socket is closed. Reconnect will be attempted in 1 second.", - event.reason, - ); - setTimeout(function () { - ws = new WebSocket(url); - }, 1000); }; return () => {