From eca5330f4480f90d1a2137e77a990217c47d8b5b Mon Sep 17 00:00:00 2001 From: Sara Date: Fri, 13 Oct 2023 19:10:15 +0200 Subject: [PATCH 01/13] adds meeting setup --- www/app/transcripts/[transcriptId]/page.tsx | 2 +- .../[transcriptId]/record/page.tsx | 146 +++++++++++ www/app/transcripts/createTranscript.ts | 54 ++++ www/app/transcripts/new/page.tsx | 237 ++++++------------ www/app/transcripts/useTranscript.ts | 50 +--- 5 files changed, 281 insertions(+), 208 deletions(-) create mode 100644 www/app/transcripts/[transcriptId]/record/page.tsx create mode 100644 www/app/transcripts/createTranscript.ts diff --git a/www/app/transcripts/[transcriptId]/page.tsx b/www/app/transcripts/[transcriptId]/page.tsx index 74fede74..6cf3b531 100644 --- a/www/app/transcripts/[transcriptId]/page.tsx +++ b/www/app/transcripts/[transcriptId]/page.tsx @@ -21,7 +21,7 @@ type TranscriptDetails = { export default function TranscriptDetails(details: TranscriptDetails) { const api = getApi(); - const transcript = useTranscript(null, api, details.params.transcriptId); + const transcript = useTranscript(details.params.transcriptId); const topics = useTopics(api, details.params.transcriptId); const waveform = useWaveform(api, details.params.transcriptId); const useActiveTopic = useState(null); diff --git a/www/app/transcripts/[transcriptId]/record/page.tsx b/www/app/transcripts/[transcriptId]/record/page.tsx new file mode 100644 index 00000000..6e17e07f --- /dev/null +++ b/www/app/transcripts/[transcriptId]/record/page.tsx @@ -0,0 +1,146 @@ +"use client"; +import React, { useEffect, useState } from "react"; +import Recorder from "../../recorder"; +import { TopicList } from "../../topicList"; +import useWebRTC from "../../useWebRTC"; +import useTranscript from "../../useTranscript"; +import { useWebSockets } from "../../useWebSockets"; +import useAudioDevice from "../../useAudioDevice"; +import "../../../styles/button.css"; +import { Topic } from "../../webSocketTypes"; +import getApi from "../../../lib/getApi"; +import LiveTrancription from "../../liveTranscription"; +import DisconnectedIndicator from "../../disconnectedIndicator"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faGear } from "@fortawesome/free-solid-svg-icons"; +import { lockWakeState, releaseWakeState } from "../../../lib/wakeLock"; + +type TranscriptDetails = { + params: { + transcriptId: string; + }; +}; + +const TranscriptRecord = (details: TranscriptDetails) => { + const [stream, setStream] = useState(null); + const [disconnected, setDisconnected] = useState(false); + const useActiveTopic = useState(null); + + useEffect(() => { + if (process.env.NEXT_PUBLIC_ENV === "development") { + document.onkeyup = (e) => { + if (e.key === "d") { + setDisconnected((prev) => !prev); + } + }; + } + }, []); + + const transcript = useTranscript(details.params.transcriptId); + const api = getApi(); + const webRTC = useWebRTC(stream, details.params.transcriptId, api); + const webSockets = useWebSockets(details.params.transcriptId); + + const { + loading, + permissionOk, + permissionDenied, + audioDevices, + requestPermission, + getAudioStream, + } = useAudioDevice(); + + const [hasRecorded, setHasRecorded] = useState(false); + const [transcriptStarted, setTranscriptStarted] = useState(false); + + useEffect(() => { + if (!transcriptStarted && webSockets.transcriptText.length !== 0) + setTranscriptStarted(true); + }, [webSockets.transcriptText]); + + useEffect(() => { + if (transcript?.response?.longSummary) { + const newUrl = `/transcripts/${transcript.response.id}`; + // Shallow redirection does not work on NextJS 13 + // https://github.com/vercel/next.js/discussions/48110 + // https://github.com/vercel/next.js/discussions/49540 + // router.push(newUrl, undefined, { shallow: true }); + history.replaceState({}, "", newUrl); + } + }); + + useEffect(() => { + lockWakeState(); + return () => { + releaseWakeState(); + }; + }, []); + + return ( + <> + { + setStream(null); + setHasRecorded(true); + webRTC?.send(JSON.stringify({ cmd: "STOP" })); + }} + topics={webSockets.topics} + getAudioStream={getAudioStream} + useActiveTopic={useActiveTopic} + isPastMeeting={false} + audioDevices={audioDevices} + /> + +
+ + +
+ {!hasRecorded ? ( + <> + {transcriptStarted && ( +

Transcription

+ )} +
+
+ {!transcriptStarted ? ( +
+ The conversation transcript will appear here shortly after + you start recording. +
+ ) : ( + + )} +
+
+ + ) : ( +
+
+ +
+

+ We are generating the final summary for you. This may take a + couple of minutes. Please do not navigate away from the page + during this time. +

+
+ )} +
+
+ + {disconnected && } + + ); +}; + +export default TranscriptRecord; diff --git a/www/app/transcripts/createTranscript.ts b/www/app/transcripts/createTranscript.ts new file mode 100644 index 00000000..32024cb4 --- /dev/null +++ b/www/app/transcripts/createTranscript.ts @@ -0,0 +1,54 @@ +import { useEffect, useState } from "react"; +import { DefaultApi, V1TranscriptsCreateRequest } from "../api/apis/DefaultApi"; +import { GetTranscript } from "../api"; +import { useError } from "../(errors)/errorContext"; +import getApi from "../lib/getApi"; + +type CreateTranscript = { + response: GetTranscript | null; + loading: boolean; + error: Error | null; + create: (params: V1TranscriptsCreateRequest["createTranscript"]) => void; +}; + +const useCreateTranscript = (): CreateTranscript => { + const [response, setResponse] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setErrorState] = useState(null); + const { setError } = useError(); + const api = getApi(); + + const create = (params: V1TranscriptsCreateRequest["createTranscript"]) => { + if (loading) return; + + setLoading(true); + const requestParameters: V1TranscriptsCreateRequest = { + createTranscript: { + name: params.name || "Weekly All-Hands", // Default + targetLanguage: params.targetLanguage || "en", // Default + }, + }; + + console.debug( + "POST - /v1/transcripts/ - Requesting new transcription creation", + requestParameters, + ); + + api + .v1TranscriptsCreate(requestParameters) + .then((result) => { + setResponse(result); + setLoading(false); + console.debug("New transcript created:", result); + }) + .catch((err) => { + setError(err); + setErrorState(err); + setLoading(false); + }); + }; + + return { response, loading, error, create }; +}; + +export default useCreateTranscript; diff --git a/www/app/transcripts/new/page.tsx b/www/app/transcripts/new/page.tsx index b4e294d6..365770b3 100644 --- a/www/app/transcripts/new/page.tsx +++ b/www/app/transcripts/new/page.tsx @@ -1,7 +1,5 @@ "use client"; import React, { useEffect, useState } from "react"; -import Recorder from "../recorder"; -import { TopicList } from "../topicList"; import useWebRTC from "../useWebRTC"; import useTranscript from "../useTranscript"; import { useWebSockets } from "../useWebSockets"; @@ -9,35 +7,37 @@ import useAudioDevice from "../useAudioDevice"; import "../../styles/button.css"; import { Topic } from "../webSocketTypes"; import getApi from "../../lib/getApi"; -import LiveTrancription from "../liveTranscription"; -import DisconnectedIndicator from "../disconnectedIndicator"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faGear } from "@fortawesome/free-solid-svg-icons"; import About from "../../(aboutAndPrivacy)/about"; import Privacy from "../../(aboutAndPrivacy)/privacy"; import { lockWakeState, releaseWakeState } from "../../lib/wakeLock"; import { useRouter } from "next/navigation"; +import createTranscript from "../createTranscript"; +import { GetTranscript } from "../../api"; +import { Router } from "next/router"; +import useCreateTranscript from "../createTranscript"; const TranscriptCreate = () => { - const [stream, setStream] = useState(null); - const [disconnected, setDisconnected] = useState(false); - const useActiveTopic = useState(null); - - useEffect(() => { - if (process.env.NEXT_PUBLIC_ENV === "development") { - document.onkeyup = (e) => { - if (e.key === "d") { - setDisconnected((prev) => !prev); - } - }; - } - }, []); - - const api = getApi(); - const transcript = useTranscript(stream, api); - const webRTC = useWebRTC(stream, transcript?.response?.id, api); - const webSockets = useWebSockets(transcript?.response?.id); + // const transcript = useTranscript(stream, api); const router = useRouter(); + const api = getApi(); + + const [name, setName] = useState(); + const nameChange = (event: React.ChangeEvent) => { + setName(event.target.value); + }; + const [targetLanguage, setTargetLanguage] = useState(); + + const createTranscript = useCreateTranscript(); + + const send = () => { + if (createTranscript.loading || permissionDenied) return; + createTranscript.create({ name, targetLanguage }); + }; + useEffect(() => { + createTranscript.response && + router.push(`/transcripts/${createTranscript.response.id}/record`); + }, [createTranscript.response]); + const { loading, permissionOk, @@ -47,151 +47,62 @@ const TranscriptCreate = () => { getAudioStream, } = useAudioDevice(); - const [hasRecorded, setHasRecorded] = useState(false); - const [transcriptStarted, setTranscriptStarted] = useState(false); - - useEffect(() => { - if (!transcriptStarted && webSockets.transcriptText.length !== 0) - setTranscriptStarted(true); - }, [webSockets.transcriptText]); - - useEffect(() => { - if (transcript?.response?.id) { - const newUrl = `/transcripts/${transcript.response.id}`; - // Shallow redirection does not work on NextJS 13 - // https://github.com/vercel/next.js/discussions/48110 - // https://github.com/vercel/next.js/discussions/49540 - // router.push(newUrl, undefined, { shallow: true }); - history.replaceState({}, "", newUrl); - } - }); - - useEffect(() => { - lockWakeState(); - return () => { - releaseWakeState(); - }; - }, []); - return ( <> - {permissionOk ? ( - <> - { - webRTC?.send(JSON.stringify({ cmd: "STOP" })); - setStream(null); - setHasRecorded(true); - }} - topics={webSockets.topics} - getAudioStream={getAudioStream} - useActiveTopic={useActiveTopic} - isPastMeeting={false} - audioDevices={audioDevices} - /> - -
- - -
- {!hasRecorded ? ( - <> - {transcriptStarted && ( -

Transcription

- )} -
-
- {!transcriptStarted ? ( -
- The conversation transcript will appear here shortly - after you start recording. -
- ) : ( - - )} -
-
- +
+
+
+
+
+

+ Welcome to reflector.media +

+

+ Reflector is a transcription and summarization pipeline that + transforms audio into knowledge. The output is meeting minutes + and topic summaries enabling topic-specific analyses stored in + your systems of record. This is accomplished on your + infrastructure – without 3rd parties – keeping your data + private, secure, and organized. +

+ + + +

+ Audio Permissions +

+ {loading ? ( +

Checking permission...

+ ) : permissionOk ? ( + <> Microphone permission granted ) : ( -
-
- -
-

- We are generating the final summary for you. This may take a - couple of minutes. Please do not navigate away from the page - during this time. + <> +

+ In order to use Reflector, we kindly request permission to + access your microphone during meetings and events. +
+ +
+ {permissionDenied + ? "Permission to use your microphone was denied, please change the permission setting in your browser and refresh this page." + : "Please grant permission to continue."}

-
+ + )} -
+
+
- - {disconnected && } - - ) : ( - <> -
-
-
-
-
-

- Welcome to reflector.media -

-

- Reflector is a transcription and summarization pipeline that - transforms audio into knowledge. The output is meeting - minutes and topic summaries enabling topic-specific analyses - stored in your systems of record. This is accomplished on - your infrastructure – without 3rd parties – keeping your - data private, secure, and organized. -

- -

- Audio Permissions -

- {loading ? ( -

Checking permission...

- ) : ( - <> -

- In order to use Reflector, we kindly request permission - to access your microphone during meetings and events. -
- -
- {permissionDenied - ? "Permission to use your microphone was denied, please change the permission setting in your browser and refresh this page." - : "Please grant permission to continue."} -

- - - )} -
-
-
-
- - )} + + ); }; diff --git a/www/app/transcripts/useTranscript.ts b/www/app/transcripts/useTranscript.ts index 7dbc969a..eb39b202 100644 --- a/www/app/transcripts/useTranscript.ts +++ b/www/app/transcripts/useTranscript.ts @@ -1,11 +1,11 @@ import { useEffect, useState } from "react"; import { - DefaultApi, V1TranscriptGetRequest, V1TranscriptsCreateRequest, } from "../api/apis/DefaultApi"; import { GetTranscript } from "../api"; import { useError } from "../(errors)/errorContext"; +import getApi from "../lib/getApi"; type Transcript = { response: GetTranscript | null; @@ -13,23 +13,12 @@ type Transcript = { error: Error | null; }; -const useTranscript = ( - stream: MediaStream | null, - api: DefaultApi, - id: string | null = null, -): Transcript => { +const useTranscript = (id: string | null): Transcript => { const [response, setResponse] = useState(null); const [loading, setLoading] = useState(false); const [error, setErrorState] = useState(null); const { setError } = useError(); - - const getOrCreateTranscript = (id: string | null) => { - if (id) { - getTranscript(id); - } else if (stream) { - createTranscript(); - } - }; + const api = getApi(); const getTranscript = (id: string | null) => { if (!id) throw new Error("Transcript ID is required to get transcript"); @@ -43,34 +32,7 @@ const useTranscript = ( .then((result) => { setResponse(result); setLoading(false); - console.debug("New transcript created:", result); - }) - .catch((err) => { - setError(err); - setErrorState(err); - }); - }; - - const createTranscript = () => { - setLoading(true); - const requestParameters: V1TranscriptsCreateRequest = { - createTranscript: { - name: "Weekly All-Hands", // Hardcoded for now - targetLanguage: "en", // Hardcoded for now - }, - }; - - console.debug( - "POST - /v1/transcripts/ - Requesting new transcription creation", - requestParameters, - ); - - api - .v1TranscriptsCreate(requestParameters) - .then((result) => { - setResponse(result); - setLoading(false); - console.debug("New transcript created:", result); + console.debug("Transcript Loaded:", result); }) .catch((err) => { setError(err); @@ -79,8 +41,8 @@ const useTranscript = ( }; useEffect(() => { - getOrCreateTranscript(id); - }, [id, stream]); + getTranscript(id); + }, [id]); return { response, loading, error }; }; From 47fc52af11909e85f602f382c9c8cff4830db01f Mon Sep 17 00:00:00 2001 From: Sara Date: Fri, 13 Oct 2023 23:35:19 +0200 Subject: [PATCH 02/13] adds three-letter language picker --- www/app/supportedLanguages.ts | 496 ++++++++++++++++++ .../[transcriptId]/record/page.tsx | 14 +- www/app/transcripts/createTranscript.ts | 2 +- www/app/transcripts/liveTranscription.tsx | 9 +- www/app/transcripts/new/page.tsx | 134 ++--- www/app/transcripts/useWebSockets.ts | 10 +- www/package.json | 1 + www/yarn.lock | 5 + 8 files changed, 593 insertions(+), 78 deletions(-) create mode 100644 www/app/supportedLanguages.ts diff --git a/www/app/supportedLanguages.ts b/www/app/supportedLanguages.ts new file mode 100644 index 00000000..6e84cec5 --- /dev/null +++ b/www/app/supportedLanguages.ts @@ -0,0 +1,496 @@ +import Script from "next/script"; + +// type Script = 'Latn' | 'Ethi' | 'Arab' | 'Beng' | 'Cyrl' | 'Taml' | 'Hant' | 'Hans' | 'Grek' | 'Gujr' | 'Hebr'| 'Deva'| 'Armn' | 'Jpan' | 'Knda' | 'Geor'; +type LanguageOption = { + value: string | undefined; + name: string; + script?: string; +}; + +const supportedLanguages: LanguageOption[] = [ + { value: "afr", name: "Afrikaans", script: "Latn" }, + { + value: "amh", + name: "Amharic", + script: "Ethi", + }, + { + value: "arb", + name: "Modern Standard Arabic", + script: "Arab", + }, + { + value: "ary", + name: "Moroccan Arabic", + script: "Arab", + }, + { + value: "arz", + name: "Egyptian Arabic", + script: "Arab", + }, + { + value: "asm", + name: "Assamese", + script: "Beng", + }, + { + value: "azj", + name: "North Azerbaijani", + script: "Latn", + }, + { + value: "bel", + name: "Belarusian", + script: "Cyrl", + }, + { + value: "ben", + name: "Bengali", + script: "Beng", + }, + { + value: "bos", + name: "Bosnian", + script: "Latn", + }, + { + value: "bul", + name: "Bulgarian", + script: "Cyrl", + }, + { + value: "cat", + name: "Catalan", + script: "Latn", + }, + { + value: "ceb", + name: "Cebuano", + script: "Latn", + }, + { + value: "ces", + name: "Czech", + script: "Latn", + }, + { + value: "ckb", + name: "Central Kurdish", + script: "Arab", + }, + { + value: "cmn", + name: "Mandarin Chinese", + script: "Hans", + }, + { + value: "cmn_Ha", + name: "Mandarin Chinese", + script: "Hant", + }, + { + value: "cym", + name: "Welsh", + script: "Latn", + }, + { + value: "dan", + name: "Danish", + script: "Latn", + }, + { + value: "deu", + name: "German", + script: "Latn", + }, + { + value: "ell", + name: "Greek", + script: "Grek", + }, + { + value: "eng", + name: "English", + script: "Latn", + }, + { + value: "est", + name: "Estonian", + script: "Latn", + }, + { + value: "eus", + name: "Basque", + script: "Latn", + }, + { + value: "fin", + name: "Finnish", + script: "Latn", + }, + { + value: "fra", + name: "French", + script: "Latn", + }, + { + value: "gaz", + name: "West Central Oromo", + script: "Latn", + }, + { + value: "gle", + name: "Irish", + script: "Latn", + }, + { + value: "glg", + name: "Galician", + script: "Latn", + }, + { + value: "guj", + name: "Gujarati", + script: "Gujr", + }, + { + value: "heb", + name: "Hebrew", + script: "Hebr", + }, + { + value: "hin", + name: "Hindi", + script: "Deva", + }, + { + value: "hrv", + name: "Croatian", + script: "Latn", + }, + { + value: "hun", + name: "Hungarian", + script: "Latn", + }, + { + value: "hye", + name: "Armenian", + script: "Armn", + }, + { + value: "ibo", + name: "Igbo", + script: "Latn", + }, + { + value: "ind", + name: "Indonesian", + script: "Latn", + }, + { + value: "isl", + name: "Icelandic", + script: "Latn", + }, + { + value: "ita", + name: "Italian", + script: "Latn", + }, + { + value: "jav", + name: "Javanese", + script: "Latn", + }, + { + value: "jpn", + name: "Japanese", + script: "Jpan", + }, + { + value: "kan", + name: "Kannada", + script: "Knda", + }, + { + value: "kat", + name: "Georgian", + script: "Geor", + }, + { + value: "kaz", + name: "Kazakh", + script: "Cyrl", + }, + { + value: "khk", + name: "Halh Mongolian", + script: "Cyrl", + }, + { + value: "khm", + name: "Khmer", + script: "Khmr", + }, + { + value: "kir", + name: "Kyrgyz", + script: "Cyrl", + }, + { + value: "kor", + name: "Korean", + script: "Kore", + }, + { + value: "lao", + name: "Lao", + script: "Laoo", + }, + { + value: "lit", + name: "Lithuanian", + script: "Latn", + }, + { + value: "lug", + name: "Ganda", + script: "Latn", + }, + { + value: "luo", + name: "Luo", + script: "Latn", + }, + { + value: "lvs", + name: "Standard Latvian", + script: "Latn", + }, + { + value: "mai", + name: "Maithili", + script: "Deva", + }, + { + value: "mal", + name: "Malayalam", + script: "Mlym", + }, + { + value: "mar", + name: "Marathi", + script: "Deva", + }, + { + value: "mkd", + name: "Macedonian", + script: "Cyrl", + }, + { + value: "mlt", + name: "Maltese", + script: "Latn", + }, + { + value: "mni", + name: "Meitei", + script: "Beng", + }, + { + value: "mya", + name: "Burmese", + script: "Mymr", + }, + { + value: "nld", + name: "Dutch", + script: "Latn", + }, + { + value: "nno", + name: "Norwegian Nynorsk", + script: "Latn", + }, + { + value: "nob", + name: "Norwegian Bokmål", + script: "Latn", + }, + { + value: "npi", + name: "Nepali", + script: "Deva", + }, + { + value: "nya", + name: "Nyanja", + script: "Latn", + }, + { + value: "ory", + name: "Odia", + script: "Orya", + }, + { + value: "pan", + name: "Punjabi", + script: "Guru", + }, + { + value: "pbt", + name: "Southern Pashto", + script: "Arab", + }, + { + value: "pes", + name: "Western Persian", + script: "Arab", + }, + { + value: "pol", + name: "Polish", + script: "Latn", + }, + { + value: "por", + name: "Portuguese", + script: "Latn", + }, + { + value: "ron", + name: "Romanian", + script: "Latn", + }, + { + value: "rus", + name: "Russian", + script: "Cyrl", + }, + { + value: "slk", + name: "Slovak", + script: "Latn", + }, + { + value: "slv", + name: "Slovenian", + script: "Latn", + }, + { + value: "sna", + name: "Shona", + script: "Latn", + }, + { + value: "snd", + name: "Sindhi", + script: "Arab", + }, + { + value: "som", + name: "Somali", + script: "Latn", + }, + { + value: "spa", + name: "Spanish", + script: "Latn", + }, + { + value: "srp", + name: "Serbian", + script: "Cyrl", + }, + { + value: "swe", + name: "Swedish", + script: "Latn", + }, + { + value: "swh", + name: "Swahili", + script: "Latn", + }, + { + value: "tam", + name: "Tamil", + script: "Taml", + }, + { + value: "tel", + name: "Telugu", + script: "Telu", + }, + { + value: "tgk", + name: "Tajik", + script: "Cyrl", + }, + { + value: "tgl", + name: "Tagalog", + script: "Latn", + }, + { + value: "tha", + name: "Thai", + script: "Thai", + }, + { + value: "tur", + name: "Turkish", + script: "Latn", + }, + { + value: "ukr", + name: "Ukrainian", + script: "Cyrl", + }, + { + value: "urd", + name: "Urdu", + script: "Arab", + }, + { + value: "uzn", + name: "Northern Uzbek", + script: "Latn", + }, + { + value: "vie", + name: "Vietnamese", + script: "Latn", + }, + { + value: "yor", + name: "Yoruba", + script: "Latn", + }, + { + value: "yue", + name: "Cantonese", + script: "Hant", + }, + { + value: "zsm", + name: "Standard Malay", + script: "Latn", + }, + { + value: "zul", + name: "Zulu", + script: "Latn", + }, +]; + +const supportedLatinLanguages = supportedLanguages.filter( + (lan) => lan.script == "Latn", +); +supportedLatinLanguages.push({ value: undefined, name: "None" }); + +export { supportedLatinLanguages }; + +export default supportedLanguages; diff --git a/www/app/transcripts/[transcriptId]/record/page.tsx b/www/app/transcripts/[transcriptId]/record/page.tsx index 6e17e07f..2e212c2e 100644 --- a/www/app/transcripts/[transcriptId]/record/page.tsx +++ b/www/app/transcripts/[transcriptId]/record/page.tsx @@ -41,14 +41,7 @@ const TranscriptRecord = (details: TranscriptDetails) => { const webRTC = useWebRTC(stream, details.params.transcriptId, api); const webSockets = useWebSockets(details.params.transcriptId); - const { - loading, - permissionOk, - permissionDenied, - audioDevices, - requestPermission, - getAudioStream, - } = useAudioDevice(); + const { audioDevices, getAudioStream } = useAudioDevice(); const [hasRecorded, setHasRecorded] = useState(false); const [transcriptStarted, setTranscriptStarted] = useState(false); @@ -115,7 +108,10 @@ const TranscriptRecord = (details: TranscriptDetails) => { you start recording. ) : ( - + )} diff --git a/www/app/transcripts/createTranscript.ts b/www/app/transcripts/createTranscript.ts index 32024cb4..d07cac51 100644 --- a/www/app/transcripts/createTranscript.ts +++ b/www/app/transcripts/createTranscript.ts @@ -25,7 +25,7 @@ const useCreateTranscript = (): CreateTranscript => { const requestParameters: V1TranscriptsCreateRequest = { createTranscript: { name: params.name || "Weekly All-Hands", // Default - targetLanguage: params.targetLanguage || "en", // Default + targetLanguage: params.targetLanguage || "eng", // Default }, }; diff --git a/www/app/transcripts/liveTranscription.tsx b/www/app/transcripts/liveTranscription.tsx index 2c913a91..6e40e7a3 100644 --- a/www/app/transcripts/liveTranscription.tsx +++ b/www/app/transcripts/liveTranscription.tsx @@ -1,14 +1,19 @@ type LiveTranscriptionProps = { text: string; + translateText: string; }; export default function LiveTrancription(props: LiveTranscriptionProps) { return (

- {/* Nous allons prendre quelques appels téléphoniques et répondre à quelques questions */} - {props.text} + {props.translateText ? props.translateText : props.text}

+ {props.translateText && ( +

+ {props.text} +

+ )}
); } diff --git a/www/app/transcripts/new/page.tsx b/www/app/transcripts/new/page.tsx index 365770b3..f2b15120 100644 --- a/www/app/transcripts/new/page.tsx +++ b/www/app/transcripts/new/page.tsx @@ -1,20 +1,15 @@ "use client"; import React, { useEffect, useState } from "react"; -import useWebRTC from "../useWebRTC"; -import useTranscript from "../useTranscript"; -import { useWebSockets } from "../useWebSockets"; import useAudioDevice from "../useAudioDevice"; import "../../styles/button.css"; -import { Topic } from "../webSocketTypes"; import getApi from "../../lib/getApi"; import About from "../../(aboutAndPrivacy)/about"; import Privacy from "../../(aboutAndPrivacy)/privacy"; -import { lockWakeState, releaseWakeState } from "../../lib/wakeLock"; import { useRouter } from "next/navigation"; -import createTranscript from "../createTranscript"; -import { GetTranscript } from "../../api"; -import { Router } from "next/router"; import useCreateTranscript from "../createTranscript"; +import SelectSearch from "react-select-search"; +import { supportedLatinLanguages } from "../../supportedLanguages"; +import "react-select-search/style.css"; const TranscriptCreate = () => { // const transcript = useTranscript(stream, api); @@ -27,6 +22,10 @@ const TranscriptCreate = () => { }; const [targetLanguage, setTargetLanguage] = useState(); + const onLanguageChange = (newval) => { + typeof newval === "string" && setTargetLanguage(newval); + }; + const createTranscript = useCreateTranscript(); const send = () => { @@ -38,70 +37,75 @@ const TranscriptCreate = () => { router.push(`/transcripts/${createTranscript.response.id}/record`); }, [createTranscript.response]); - const { - loading, - permissionOk, - permissionDenied, - audioDevices, - requestPermission, - getAudioStream, - } = useAudioDevice(); + const { loading, permissionOk, permissionDenied, requestPermission } = + useAudioDevice(); return ( <>
-
+
-
-
-

- Welcome to reflector.media -

-

- Reflector is a transcription and summarization pipeline that - transforms audio into knowledge. The output is meeting minutes - and topic summaries enabling topic-specific analyses stored in - your systems of record. This is accomplished on your - infrastructure – without 3rd parties – keeping your data - private, secure, and organized. -

- - - -

- Audio Permissions -

- {loading ? ( -

Checking permission...

- ) : permissionOk ? ( - <> Microphone permission granted - ) : ( - <> -

- In order to use Reflector, we kindly request permission to - access your microphone during meetings and events. -
- -
- {permissionDenied - ? "Permission to use your microphone was denied, please change the permission setting in your browser and refresh this page." - : "Please grant permission to continue."} -

- - - )} -
- +
+

+ Welcome to reflector.media +

+

+ Reflector is a transcription and summarization pipeline that + transforms audio into knowledge. The output is meeting minutes and + topic summaries enabling topic-specific analyses stored in your + systems of record. This is accomplished on your infrastructure – + without 3rd parties – keeping your data private, secure, and + organized. +

+
+
+

Try Reflector

+ + + + + {loading ? ( +

Checking permission...

+ ) : permissionOk ? ( + <> Microphone permission granted + ) : ( + <> +

+ In order to use Reflector, we kindly request permission to + access your microphone during meetings and events. +
+ +
+ {permissionDenied && + "Permission to use your microphone was denied, please change the permission setting in your browser and refresh this page."} +

+ + + )} + +
); diff --git a/www/app/transcripts/useWebSockets.ts b/www/app/transcripts/useWebSockets.ts index cb31c4bf..a9e0cf57 100644 --- a/www/app/transcripts/useWebSockets.ts +++ b/www/app/transcripts/useWebSockets.ts @@ -5,6 +5,7 @@ import { useRouter } from "next/navigation"; type UseWebSockets = { transcriptText: string; + translateText: string; topics: Topic[]; finalSummary: FinalSummary; status: Status; @@ -12,7 +13,9 @@ type UseWebSockets = { export const useWebSockets = (transcriptId: string | null): UseWebSockets => { const [transcriptText, setTranscriptText] = useState(""); + const [translateText, setTranslateText] = useState(""); const [textQueue, setTextQueue] = useState([]); + const [translationQueue, setTranslationQueue] = useState([]); const [isProcessing, setIsProcessing] = useState(false); const [topics, setTopics] = useState([]); const [finalSummary, setFinalSummary] = useState({ @@ -30,6 +33,8 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { setIsProcessing(true); const text = textQueue[0]; setTranscriptText(text); + setTranslateText(translationQueue[0]); + console.log("displaying " + translateText); const WPM_READING = 200 + textQueue.length * 10; // words per minute to read const wordCount = text.split(/\s+/).length; @@ -38,6 +43,7 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { setTimeout(() => { setIsProcessing(false); setTextQueue((prevQueue) => prevQueue.slice(1)); + setTranslationQueue((prevQueue) => prevQueue.slice(1)); }, delay); }, [textQueue, isProcessing]); @@ -158,11 +164,13 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { switch (message.event) { case "TRANSCRIPT": const newText = (message.data.text ?? "").trim(); + const newTranslation = (message.data.translation ?? "").trim(); if (!newText) break; console.debug("TRANSCRIPT event:", newText); setTextQueue((prevQueue) => [...prevQueue, newText]); + setTranslationQueue((prevQueue) => [...prevQueue, newTranslation]); break; case "TOPIC": @@ -233,5 +241,5 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { }; }, [transcriptId]); - return { transcriptText, topics, finalSummary, status }; + return { transcriptText, translateText, topics, finalSummary, status }; }; diff --git a/www/package.json b/www/package.json index 407f5512..91de6ef7 100644 --- a/www/package.json +++ b/www/package.json @@ -27,6 +27,7 @@ "react-dropdown": "^1.11.0", "react-markdown": "^9.0.0", "react-qr-code": "^2.0.12", + "react-select-search": "^4.1.7", "sass": "^1.63.6", "simple-peer": "^9.11.1", "superagent": "^8.0.9", diff --git a/www/yarn.lock b/www/yarn.lock index 5902f7a7..d3d39205 100644 --- a/www/yarn.lock +++ b/www/yarn.lock @@ -2088,6 +2088,11 @@ react-qr-code@^2.0.12: prop-types "^15.8.1" qr.js "0.0.0" +react-select-search@^4.1.7: + version "4.1.7" + resolved "https://registry.yarnpkg.com/react-select-search/-/react-select-search-4.1.7.tgz#5662729b9052282bde52e1352006d495d9c5ed6e" + integrity sha512-pU7ONAdK+bmz2tbhBWYQv9m5mnXOn8yImuiy+5UhimIG80d5iKv3nSYJIjJWjDbdrrdoXiCRwQm8xbA8llTjmQ== + react@^18.2.0: version "18.2.0" resolved "https://registry.npmjs.org/react/-/react-18.2.0.tgz" From 90c6824f52590af5ba0e2fd22138975ee26c1682 Mon Sep 17 00:00:00 2001 From: Sara Date: Fri, 13 Oct 2023 23:36:02 +0200 Subject: [PATCH 03/13] replace two letter codes with three letter codes --- server/gpu/modal/reflector_transcriber.py | 2 +- server/gpu/modal/reflector_translator.py | 4 +- .../processors/audio_transcript_banana.py | 7 +- .../processors/audio_transcript_modal.py | 6 +- .../processors/audio_transcript_whisper.py | 4 +- .../processors/transcript_translator.py | 4 +- server/reflector/processors/types.py | 153 ++++++------------ server/reflector/tools/process.py | 9 +- server/reflector/views/rtc_offer.py | 4 +- server/reflector/views/transcripts.py | 12 +- server/tests/conftest.py | 2 +- server/tests/test_transcripts_translation.py | 24 +-- 12 files changed, 91 insertions(+), 140 deletions(-) diff --git a/server/gpu/modal/reflector_transcriber.py b/server/gpu/modal/reflector_transcriber.py index 69558c8e..5490906e 100644 --- a/server/gpu/modal/reflector_transcriber.py +++ b/server/gpu/modal/reflector_transcriber.py @@ -178,7 +178,7 @@ def web(): @app.post("/transcribe", dependencies=[Depends(apikey_auth)]) async def transcribe( file: UploadFile, - source_language: Annotated[str, Body(...)] = "en", + source_language: Annotated[str, Body(...)] = "eng", timestamp: Annotated[float, Body()] = 0.0 ) -> TranscriptResponse: audio_data = await file.read() diff --git a/server/gpu/modal/reflector_translator.py b/server/gpu/modal/reflector_translator.py index 69ea719a..22b76495 100644 --- a/server/gpu/modal/reflector_translator.py +++ b/server/gpu/modal/reflector_translator.py @@ -219,8 +219,8 @@ def web(): @app.post("/translate", dependencies=[Depends(apikey_auth)]) async def translate( text: str, - source_language: Annotated[str, Body(...)] = "en", - target_language: Annotated[str, Body(...)] = "fr", + source_language: Annotated[str, Body(...)] = "eng", + target_language: Annotated[str, Body(...)] = "fra", ) -> TranslateResponse: func = translatorstub.translate_text.spawn( text=text, diff --git a/server/reflector/processors/audio_transcript_banana.py b/server/reflector/processors/audio_transcript_banana.py index af8f647d..a33c8eb8 100644 --- a/server/reflector/processors/audio_transcript_banana.py +++ b/server/reflector/processors/audio_transcript_banana.py @@ -8,20 +8,21 @@ API will be a POST request to TRANSCRIPT_URL: "audio_url": "https://...", "audio_ext": "wav", "timestamp": 123.456 - "language": "en" + "language": "eng" } ``` """ +from pathlib import Path + +import httpx from reflector.processors.audio_transcript import AudioTranscriptProcessor from reflector.processors.audio_transcript_auto import AudioTranscriptAutoProcessor from reflector.processors.types import AudioFile, Transcript, Word from reflector.settings import settings from reflector.storage import Storage from reflector.utils.retry import retry -from pathlib import Path -import httpx class AudioTranscriptBananaProcessor(AudioTranscriptProcessor): diff --git a/server/reflector/processors/audio_transcript_modal.py b/server/reflector/processors/audio_transcript_modal.py index 201ed9d4..55e0087c 100644 --- a/server/reflector/processors/audio_transcript_modal.py +++ b/server/reflector/processors/audio_transcript_modal.py @@ -5,8 +5,8 @@ API will be a POST request to TRANSCRIPT_URL: ```form "timestamp": 123.456 -"source_language": "en" -"target_language": "en" +"source_language": "eng" +"target_language": "eng" "file":