selfhostyourtech/reflector
2
0
Fork 0
mirror of https://github.com/Monadical-SAS/reflector.git synced 2025-12-20 20:29:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

render transcript

Browse Source
This commit is contained in:
Sara
2024-02-08 20:15:13 +01:00
parent d09b0ae76a
commit 6acb2f6088
7 changed files with 264 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
www/app/[domain]/transcripts/[transcriptId]/record/page.tsx
View File

@@ -2,21 +2,16 @@
import React, { useEffect, useState } from "react"; import React, { useEffect, useState } from "react";
import Recorder from "../../recorder"; import Recorder from "../../recorder";
import { TopicList } from "../../topicList"; import { TopicList } from "../../topicList";
import useWebRTC from "../../useWebRTC";
import useTranscript from "../../useTranscript"; import useTranscript from "../../useTranscript";
import { useWebSockets } from "../../useWebSockets"; import { useWebSockets } from "../../useWebSockets";
import useAudioDevice from "../../useAudioDevice";
import "../../../../styles/button.css"; import "../../../../styles/button.css";
import { Topic } from "../../webSocketTypes"; import { Topic } from "../../webSocketTypes";
import LiveTrancription from "../../liveTranscription";
import DisconnectedIndicator from "../../disconnectedIndicator";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faGear } from "@fortawesome/free-solid-svg-icons";
import { lockWakeState, releaseWakeState } from "../../../../lib/wakeLock"; import { lockWakeState, releaseWakeState } from "../../../../lib/wakeLock";
import { useRouter } from "next/navigation"; import { useRouter } from "next/navigation";
import Player from "../../player"; import Player from "../../player";
import useMp3 from "../../useMp3"; import useMp3 from "../../useMp3";
import WaveformLoading from "../../waveformLoading"; import WaveformLoading from "../../waveformLoading";
import { Box, Grid } from "@chakra-ui/react";
type TranscriptDetails = { type TranscriptDetails = {
params: { params: {
@@ -25,59 +20,36 @@ type TranscriptDetails = {
}; };
const TranscriptRecord = (details: TranscriptDetails) => { const TranscriptRecord = (details: TranscriptDetails) => {
const [stream, setStream] = useState<MediaStream | null>(null); const transcript = useTranscript(details.params.transcriptId);
const [disconnected, setDisconnected] = useState<boolean>(false);
const useActiveTopic = useState<Topic | null>(null); const useActiveTopic = useState<Topic | null>(null);
useEffect(() => {
if (process.env.NEXT_PUBLIC_ENV === "development") {
document.onkeyup = (e) => {
if (e.key === "d") {
setDisconnected((prev) => !prev);
}
};
}
}, []);
const transcript = useTranscript(details.params.transcriptId);
const webRTC = useWebRTC(stream, details.params.transcriptId);
const webSockets = useWebSockets(details.params.transcriptId); const webSockets = useWebSockets(details.params.transcriptId);
const { audioDevices, getAudioStream } = useAudioDevice();
const [recordedTime, setRecordedTime] = useState(0);
const [startTime, setStartTime] = useState(0);
const [transcriptStarted, setTranscriptStarted] = useState(false);
let mp3 = useMp3(details.params.transcriptId, true); let mp3 = useMp3(details.params.transcriptId, true);
const router = useRouter(); const router = useRouter();
useEffect(() => { const [status, setStatus] = useState(
if (!transcriptStarted && webSockets.transcriptText.length !== 0) webSockets.status.value || transcript.response?.status || "idle",
setTranscriptStarted(true); );
}, [webSockets.transcriptText]);
useEffect(() => { useEffect(() => {
const statusToRedirect = ["ended", "error"]; //TODO HANDLE ERROR STATUS BETTER
const newStatus =
webSockets.status.value || transcript.response?.status || "idle";
setStatus(newStatus);
if (newStatus && (newStatus == "ended" || newStatus == "error")) {
console.log(newStatus, "redirecting");
//TODO if has no topic and is error, get back to new
if (
transcript.response?.status &&
(statusToRedirect.includes(transcript.response?.status) ||
statusToRedirect.includes(webSockets.status.value))
) {
const newUrl = "/transcripts/" + details.params.transcriptId; const newUrl = "/transcripts/" + details.params.transcriptId;
// Shallow redirection does not work on NextJS 13
// https://github.com/vercel/next.js/discussions/48110
// https://github.com/vercel/next.js/discussions/49540
router.replace(newUrl); router.replace(newUrl);
// history.replaceState({}, "", newUrl); }
} // history.replaceState({}, "", newUrl);
}, [webSockets.status.value, transcript.response?.status]); }, [webSockets.status.value, transcript.response?.status]);
useEffect(() => { useEffect(() => {
if (transcript.response?.status === "ended") mp3.getNow(); if (webSockets.waveform && webSockets.waveform) mp3.getNow();
}, [transcript.response]); }, [webSockets.waveform, webSockets.duration]);
useEffect(() => { useEffect(() => {
lockWakeState(); lockWakeState();
@@ -87,8 +59,31 @@ const TranscriptRecord = (details: TranscriptDetails) => {
}, []); }, []);
return ( return (
<div className="grid grid-rows-layout-topbar gap-2 lg:gap-4 max-h-full h-full"> <Grid
{webSockets.waveform && webSockets.duration && mp3?.media ? ( templateColumns="1fr"
templateRows="minmax(0, 1fr) auto"
gap={4}
mb={4}
>
<Box
padding={4}
background="gray.bg"
borderColor={"gray.bg"}
borderRadius={8}
>
<TopicList
topics={webSockets.topics}
useActiveTopic={useActiveTopic}
autoscroll={true}
transcriptId={details.params.transcriptId}
status={status}
currentTranscriptText={webSockets.accumulatedText}
/>
</Box>
{status == "processing" && // todo send an event when the mp3 is ready
webSockets.waveform &&
webSockets.duration &&
mp3?.media ? (
<Player <Player
topics={webSockets.topics || []} topics={webSockets.topics || []}
useActiveTopic={useActiveTopic} useActiveTopic={useActiveTopic}
@@ -96,78 +91,13 @@ const TranscriptRecord = (details: TranscriptDetails) => {
media={mp3.media} media={mp3.media}
mediaDuration={webSockets.duration} mediaDuration={webSockets.duration}
/> />
) : recordedTime ? ( ) : status == "processing" ? (
<WaveformLoading /> <WaveformLoading />
) : ( ) : (
<Recorder // todo: only start recording animation when you get "recorded" status
setStream={setStream} <Recorder transcriptId={details.params.transcriptId} status={status} />
onStop={() => {
setStream(null);
setRecordedTime(Date.now() - startTime);
webRTC?.send(JSON.stringify({ cmd: "STOP" }));
}}
onRecord={() => {
setStartTime(Date.now());
}}
getAudioStream={getAudioStream}
audioDevices={audioDevices}
transcriptId={details.params.transcriptId}
/>
)} )}
</Grid>
<div className="grid grid-cols-1 lg:grid-cols-2 grid-rows-mobile-inner lg:grid-rows-1 gap-2 lg:gap-4 h-full">
<TopicList
topics={webSockets.topics}
useActiveTopic={useActiveTopic}
autoscroll={true}
transcriptId={details.params.transcriptId}
/>
<section
className={`w-full h-full bg-blue-400/20 rounded-lg md:rounded-xl p-2 md:px-4`}
>
{!recordedTime ? (
<>
{transcriptStarted && (
<h2 className="md:text-lg font-bold">Transcription</h2>
)}
<div className="flex flex-col justify-center align center text-center h-full">
<div className="py-2 h-auto">
{!transcriptStarted ? (
<div className="text-center text-gray-500">
The conversation transcript will appear here shortly after
you start recording.
</div>
) : (
<LiveTrancription
text={webSockets.transcriptText}
translateText={webSockets.translateText}
/>
)}
</div>
</div>
</>
) : (
<div className="flex flex-col justify-center align center text-center h-full text-gray-500">
<div className="p-2 md:p-4">
<FontAwesomeIcon
icon={faGear}
className="animate-spin-slow h-14 w-14 md:h-20 md:w-20"
/>
</div>
<p>
We are generating the final summary for you. This may take a
couple of minutes. Please do not navigate away from the page
during this time.
</p>
{/* NTH If login required remove last sentence */}
</div>
)}
</section>
</div>
{disconnected && <DisconnectedIndicator />}
</div>
); );
}; };

2
www/app/[domain]/transcripts/audioInputsDropdown.tsx
View File

@@ -1,4 +1,4 @@
import React, { useEffect, useState } from "react"; import React from "react";
import Dropdown, { Option } from "react-dropdown"; import Dropdown, { Option } from "react-dropdown";
import "react-dropdown/style.css"; import "react-dropdown/style.css";

11
www/app/[domain]/transcripts/fileUploadButton.tsx
View File

@@ -1,9 +1,10 @@
import React from "react"; import React from "react";
import useApi from "../../lib/useApi"; import useApi from "../../lib/useApi";
import { Body_transcript_record_upload_v1_transcripts__transcript_id__record_upload_post } from "../../api"; import { Button } from "@chakra-ui/react";
type FileUploadButton = { type FileUploadButton = {
transcriptId: string; transcriptId: string;
disabled?: boolean;
}; };
export default function FileUploadButton(props: FileUploadButton) { export default function FileUploadButton(props: FileUploadButton) {
@@ -32,12 +33,14 @@ export default function FileUploadButton(props: FileUploadButton) {
return ( return (
<> <>
<button <Button
className="bg-blue-400 hover:bg-blue-500 focus-visible:bg-blue-500 text-white ml-2 md:ml:4 md:h-[78px] md:min-w-[100px] text-lg"
onClick={triggerFileUpload} onClick={triggerFileUpload}
colorScheme="blue"
mr={2}
disabled={props.disabled}
> >
Upload File Upload File
</button> </Button>
<input <input
type="file" type="file"

236
www/app/[domain]/transcripts/recorder.tsx
View File

@@ -3,39 +3,50 @@ import React, { useRef, useEffect, useState } from "react";
import WaveSurfer from "wavesurfer.js"; import WaveSurfer from "wavesurfer.js";
import RecordPlugin from "../../lib/custom-plugins/record"; import RecordPlugin from "../../lib/custom-plugins/record";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faMicrophone } from "@fortawesome/free-solid-svg-icons";
import { formatTime } from "../../lib/time"; import { formatTime } from "../../lib/time";
import AudioInputsDropdown from "./audioInputsDropdown";
import { Option } from "react-dropdown";
import { waveSurferStyles } from "../../styles/recorder"; import { waveSurferStyles } from "../../styles/recorder";
import { useError } from "../../(errors)/errorContext"; import { useError } from "../../(errors)/errorContext";
import FileUploadButton from "./fileUploadButton"; import FileUploadButton from "./fileUploadButton";
import useWebRTC from "./useWebRTC";
import useAudioDevice from "./useAudioDevice";
import {
Box,
Flex,
IconButton,
Menu,
MenuButton,
MenuItemOption,
MenuList,
MenuOptionGroup,
} from "@chakra-ui/react";
import StopRecordIcon from "../../styles/icons/stopRecord";
import PlayIcon from "../../styles/icons/play";
import { LuScreenShare } from "react-icons/lu";
import { FaMicrophone } from "react-icons/fa";
type RecorderProps = { type RecorderProps = {
setStream: React.Dispatch<React.SetStateAction<MediaStream | null>>;
onStop: () => void;
onRecord?: () => void;
getAudioStream: (deviceId) => Promise<MediaStream | null>;
audioDevices: Option[];
transcriptId: string; transcriptId: string;
status: string;
}; };
export default function Recorder(props: RecorderProps) { export default function Recorder(props: RecorderProps) {
const waveformRef = useRef<HTMLDivElement>(null); const waveformRef = useRef<HTMLDivElement>(null);
const [wavesurfer, setWavesurfer] = useState<WaveSurfer | null>(null);
const [record, setRecord] = useState<RecordPlugin | null>(null); const [record, setRecord] = useState<RecordPlugin | null>(null);
const [isRecording, setIsRecording] = useState<boolean>(false); const [isRecording, setIsRecording] = useState<boolean>(false);
const [hasRecorded, setHasRecorded] = useState<boolean>(false);
const [isPlaying, setIsPlaying] = useState<boolean>(false);
const [currentTime, setCurrentTime] = useState<number>(0);
const [timeInterval, setTimeInterval] = useState<number | null>(null);
const [duration, setDuration] = useState<number>(0); const [duration, setDuration] = useState<number>(0);
const [deviceId, setDeviceId] = useState<string | null>(null); const [deviceId, setDeviceId] = useState<string | null>(null);
const [recordStarted, setRecordStarted] = useState(false);
const [showDevices, setShowDevices] = useState(false);
const { setError } = useError(); const { setError } = useError();
const [stream, setStream] = useState<MediaStream | null>(null);
// Time tracking, iirc it was drifting without this. to be tested again.
const [startTime, setStartTime] = useState(0);
const [currentTime, setCurrentTime] = useState<number>(0);
const [timeInterval, setTimeInterval] = useState<number | null>(null);
const webRTC = useWebRTC(stream, props.transcriptId);
const { audioDevices, getAudioStream } = useAudioDevice();
// Function used to setup keyboard shortcuts for the streamdeck // Function used to setup keyboard shortcuts for the streamdeck
const setupProjectorKeys = (): (() => void) => { const setupProjectorKeys = (): (() => void) => {
@@ -106,22 +117,13 @@ export default function Recorder(props: RecorderProps) {
waveSurferStyles.playerStyle.backgroundColor; waveSurferStyles.playerStyle.backgroundColor;
wsWrapper.style.borderRadius = waveSurferStyles.playerStyle.borderRadius; wsWrapper.style.borderRadius = waveSurferStyles.playerStyle.borderRadius;
_wavesurfer.on("play", () => {
setIsPlaying(true);
});
_wavesurfer.on("pause", () => {
setIsPlaying(false);
});
_wavesurfer.on("timeupdate", setCurrentTime); _wavesurfer.on("timeupdate", setCurrentTime);
setRecord(_wavesurfer.registerPlugin(RecordPlugin.create())); setRecord(_wavesurfer.registerPlugin(RecordPlugin.create()));
setWavesurfer(_wavesurfer);
return () => { return () => {
_wavesurfer.destroy(); _wavesurfer.destroy();
setIsRecording(false); setIsRecording(false);
setIsPlaying(false);
setCurrentTime(0); setCurrentTime(0);
}; };
} }
@@ -130,7 +132,7 @@ export default function Recorder(props: RecorderProps) {
useEffect(() => { useEffect(() => {
if (isRecording) { if (isRecording) {
const interval = window.setInterval(() => { const interval = window.setInterval(() => {
setCurrentTime((prev) => prev + 1); setCurrentTime(Date.now() - startTime);
}, 1000); }, 1000);
setTimeInterval(interval); setTimeInterval(interval);
return () => clearInterval(interval); return () => clearInterval(interval);
@@ -147,20 +149,20 @@ export default function Recorder(props: RecorderProps) {
if (!record) return console.log("no record"); if (!record) return console.log("no record");
if (record.isRecording()) { if (record.isRecording()) {
if (props.onStop) props.onStop(); setStream(null);
webRTC?.send(JSON.stringify({ cmd: "STOP" }));
record.stopRecording(); record.stopRecording();
if (screenMediaStream) { if (screenMediaStream) {
screenMediaStream.getTracks().forEach((t) => t.stop()); screenMediaStream.getTracks().forEach((t) => t.stop());
} }
setIsRecording(false); setIsRecording(false);
setHasRecorded(true);
setScreenMediaStream(null); setScreenMediaStream(null);
setDestinationStream(null); setDestinationStream(null);
} else { } else {
if (props.onRecord) props.onRecord(); const stream = await getMicrophoneStream();
const stream = await getCurrentStream(); setStartTime(Date.now());
if (props.setStream) props.setStream(stream); setStream(stream);
if (stream) { if (stream) {
await record.startRecording(stream); await record.startRecording(stream);
setIsRecording(true); setIsRecording(true);
@@ -198,7 +200,7 @@ export default function Recorder(props: RecorderProps) {
if (destinationStream !== null) return console.log("already recording"); if (destinationStream !== null) return console.log("already recording");
// connect mic audio (microphone) // connect mic audio (microphone)
const micStream = await getCurrentStream(); const micStream = await getMicrophoneStream();
if (!micStream) { if (!micStream) {
console.log("no microphone audio"); console.log("no microphone audio");
return; return;
@@ -227,7 +229,7 @@ export default function Recorder(props: RecorderProps) {
useEffect(() => { useEffect(() => {
if (!record) return; if (!record) return;
if (!destinationStream) return; if (!destinationStream) return;
if (props.setStream) props.setStream(destinationStream); setStream(destinationStream);
if (destinationStream) { if (destinationStream) {
record.startRecording(destinationStream); record.startRecording(destinationStream);
setIsRecording(true); setIsRecording(true);
@@ -238,115 +240,87 @@ export default function Recorder(props: RecorderProps) {
startTabRecording(); startTabRecording();
}, [record, screenMediaStream]); }, [record, screenMediaStream]);
const handlePlayClick = () => {
wavesurfer?.playPause();
};
const timeLabel = () => { const timeLabel = () => {
if (isRecording) return formatTime(currentTime); if (isRecording) return formatTime(currentTime);
if (duration) return `${formatTime(currentTime)}/${formatTime(duration)}`; if (duration) return `${formatTime(currentTime)}/${formatTime(duration)}`;
return ""; return "";
}; };
const getCurrentStream = async () => { const getMicrophoneStream = async () => {
setRecordStarted(true); return deviceId && getAudioStream ? await getAudioStream(deviceId) : null;
return deviceId && props.getAudioStream
? await props.getAudioStream(deviceId)
: null;
}; };
useEffect(() => { useEffect(() => {
if (props.audioDevices && props.audioDevices.length > 0) { if (audioDevices && audioDevices.length > 0) {
setDeviceId(props.audioDevices[0].value); setDeviceId(audioDevices[0].value);
} }
}, [props.audioDevices]); }, [audioDevices]);
return ( return (
<div className="flex items-center w-full relative"> <Flex className="flex items-center w-full relative">
<div className="flex-grow items-end relative"> <IconButton
<div aria-label={isRecording ? "Stop" : "Record"}
ref={waveformRef} icon={isRecording ? <StopRecordIcon /> : <PlayIcon />}
className="flex-grow rounded-lg md:rounded-xl h-20" variant={"ghost"}
></div> colorScheme={"blue"}
<div className="absolute right-2 bottom-0"> mr={2}
{isRecording && ( onClick={handleRecClick}
<div className="inline-block bg-red-500 rounded-full w-2 h-2 my-auto mr-1 animate-ping"></div> />
)} <FileUploadButton
transcriptId={props.transcriptId}
disabled={isRecording}
></FileUploadButton>
{!isRecording && (window as any).chrome && (
<IconButton
aria-label={"Record Tab"}
icon={<LuScreenShare />}
variant={"ghost"}
colorScheme={"blue"}
disabled={isRecording}
mr={2}
onClick={handleRecordTabClick}
/>
)}
{audioDevices && audioDevices?.length > 0 && deviceId && !isRecording && (
<Menu>
<MenuButton
as={IconButton}
aria-label={"Switch microphone"}
icon={<FaMicrophone />}
variant={"ghost"}
disabled={isRecording}
colorScheme={"blue"}
mr={2}
/>
<MenuList>
<MenuOptionGroup defaultValue={audioDevices[0].value} type="radio">
{audioDevices.map((device) => (
<MenuItemOption
key={device.value}
value={device.value}
onClick={() => setDeviceId(device.value)}
>
{device.label}
</MenuItemOption>
))}
</MenuOptionGroup>
</MenuList>
</Menu>
)}
<Box position="relative" flex={1}>
<Box ref={waveformRef} height={14}></Box>
<Box
zIndex={50}
backgroundColor="rgba(255, 255, 255, 0.5)"
fontSize={"sm"}
shadow={"0px 0px 4px 0px white"}
position={"absolute"}
right={0}
bottom={0}
>
{timeLabel()} {timeLabel()}
</div> </Box>
</div> </Box>
</Flex>
{hasRecorded && (
<>
<button
className={`${
isPlaying
? "bg-orange-400 hover:bg-orange-500 focus-visible:bg-orange-500"
: "bg-green-400 hover:bg-green-500 focus-visible:bg-green-500"
} text-white ml-2 md:ml:4 md:h-[78px] md:min-w-[100px] text-lg`}
id="play-btn"
onClick={handlePlayClick}
>
{isPlaying ? "Pause" : "Play"}
</button>
</>
)}
{!hasRecorded && (
<>
<button
className={`${
isRecording
? "bg-red-400 hover:bg-red-500 focus-visible:bg-red-500"
: "bg-blue-400 hover:bg-blue-500 focus-visible:bg-blue-500"
} text-white ml-2 md:ml:4 md:h-[78px] md:min-w-[100px] text-lg`}
onClick={handleRecClick}
disabled={isPlaying}
>
{isRecording ? "Stop" : "Record"}
</button>
<FileUploadButton
transcriptId={props.transcriptId}
></FileUploadButton>
{!isRecording && (
<button
className={`${
isRecording
? "bg-red-400 hover:bg-red-500 focus-visible:bg-red-500"
: "bg-blue-400 hover:bg-blue-500 focus-visible:bg-blue-500"
} text-white ml-2 md:ml:4 md:h-[78px] md:min-w-[100px] text-lg`}
onClick={handleRecordTabClick}
>
Record
<br />a tab
</button>
)}
{props.audioDevices && props.audioDevices?.length > 0 && deviceId && (
<>
<button
className="text-center text-blue-400 hover:text-blue-700 ml-2 md:ml:4 p-2 rounded-lg focus-visible:outline outline-blue-400"
onClick={() => setShowDevices((prev) => !prev)}
>
<FontAwesomeIcon icon={faMicrophone} className="h-5 w-auto" />
</button>
<div
className={`absolute z-20 bottom-[-1rem] right-0 bg-white rounded ${
showDevices ? "visible" : "invisible"
}`}
>
<AudioInputsDropdown
setDeviceId={setDeviceId}
audioDevices={props.audioDevices}
disabled={recordStarted}
hide={() => setShowDevices(false)}
deviceId={deviceId}
/>
</div>
</>
)}
</>
)}
</div>
); );
} }

101
www/app/[domain]/transcripts/topicList.tsx
View File

@@ -14,6 +14,7 @@ import {
Flex, Flex,
Text, Text,
} from "@chakra-ui/react"; } from "@chakra-ui/react";
import { featureEnabled } from "../domainContext";
type TopicListProps = { type TopicListProps = {
topics: Topic[]; topics: Topic[];
@@ -23,6 +24,8 @@ type TopicListProps = {
]; ];
autoscroll: boolean; autoscroll: boolean;
transcriptId: string; transcriptId: string;
status: string;
currentTranscriptText: any;
}; };
export function TopicList({ export function TopicList({
@@ -30,6 +33,8 @@ export function TopicList({
useActiveTopic, useActiveTopic,
autoscroll, autoscroll,
transcriptId, transcriptId,
status,
currentTranscriptText,
}: TopicListProps) { }: TopicListProps) {
const [activeTopic, setActiveTopic] = useActiveTopic; const [activeTopic, setActiveTopic] = useActiveTopic;
const [autoscrollEnabled, setAutoscrollEnabled] = useState<boolean>(true); const [autoscrollEnabled, setAutoscrollEnabled] = useState<boolean>(true);
@@ -72,7 +77,7 @@ export function TopicList({
useEffect(() => { useEffect(() => {
if (autoscroll) { if (autoscroll) {
const topicsDiv = document.getElementById("topics-div"); const topicsDiv = document.getElementById("scroll-div");
topicsDiv && toggleScroll(topicsDiv); topicsDiv && toggleScroll(topicsDiv);
} }
@@ -80,10 +85,10 @@ export function TopicList({
useEffect(() => { useEffect(() => {
if (autoscroll && autoscrollEnabled) scrollToBottom(); if (autoscroll && autoscrollEnabled) scrollToBottom();
}, [topics.length]); }, [topics.length, currentTranscriptText]);
const scrollToBottom = () => { const scrollToBottom = () => {
const topicsDiv = document.getElementById("topics-div"); const topicsDiv = document.getElementById("scroll-div");
if (topicsDiv) topicsDiv.scrollTop = topicsDiv.scrollHeight; if (topicsDiv) topicsDiv.scrollTop = topicsDiv.scrollHeight;
}; };
@@ -97,30 +102,41 @@ export function TopicList({
); );
}; };
const requireLogin = featureEnabled("requireLogin");
useEffect(() => {
setActiveTopic(topics[topics.length - 1]);
}, [topics]);
useEffect(() => {
if (activeTopic && currentTranscriptText) setActiveTopic(null);
}, [activeTopic, currentTranscriptText]);
return ( return (
<Flex <Flex
position={"relative"} position={"relative"}
w={"100%"} w={"100%"}
h={"100%"} h={"100%"}
dir="column" flexDirection={"column"}
justify={"center"} justify={"center"}
align={"center"} align={"center"}
flexShrink={0} flexShrink={0}
> >
{topics.length > 0 ? ( {autoscroll && (
<> <ScrollToBottom
{autoscroll && ( visible={!autoscrollEnabled}
<ScrollToBottom handleScrollBottom={scrollToBottom}
visible={!autoscrollEnabled} />
handleScrollBottom={scrollToBottom} )}
/>
)}
<Box
id="scroll-div"
overflowY={"auto"}
h={"100%"}
onScroll={handleScroll}
>
{topics.length > 0 && (
<Accordion <Accordion
id="topics-div"
overflowY={"auto"}
h={"100%"}
onScroll={handleScroll}
index={topics.findIndex((topic) => topic.id == activeTopic?.id)} index={topics.findIndex((topic) => topic.id == activeTopic?.id)}
variant="custom" variant="custom"
allowToggle allowToggle
@@ -200,18 +216,47 @@ export function TopicList({
</AccordionItem> </AccordionItem>
))} ))}
</Accordion> </Accordion>
</> )}
) : (
<Box textAlign={"center"} textColor="gray"> {status == "recording" && (
<Text> <Box textAlign={"center"}>
Discussion topics will appear here after you start recording. <Text>{currentTranscriptText}</Text>
</Text> </Box>
<Text> )}
It may take up to 5 minutes of conversation for the first topic to {(status == "recording" || status == "idle") &&
appear. currentTranscriptText.length == 0 &&
</Text> topics.length == 0 && (
</Box> <Box textAlign={"center"} textColor="gray">
)} <Text>
Discussion transcript will appear here after you start
recording.
</Text>
<Text>
It may take up to 5 minutes of conversation to first appear.
</Text>
</Box>
)}
{status == "processing" && (
<Box textAlign={"center"} textColor="gray">
<Text>We are processing the recording, please wait.</Text>
{!requireLogin && (
<span>
Please do not navigate away from the page during this time.
</span>
)}
</Box>
)}
{status == "ended" && topics.length == 0 && (
<Box textAlign={"center"} textColor="gray">
<Text>Recording has ended without topics being found.</Text>
</Box>
)}
{status == "error" && (
<Box textAlign={"center"} textColor="gray">
<Text>There was an error processing your recording</Text>
</Box>
)}
</Box>
</Flex> </Flex>
); );
} }

37
www/app/[domain]/transcripts/useWebSockets.ts
View File

@@ -6,8 +6,9 @@ import { AudioWaveform, GetTranscriptSegmentTopic } from "../../api";
import useApi from "../../lib/useApi"; import useApi from "../../lib/useApi";
export type UseWebSockets = { export type UseWebSockets = {
transcriptText: string; transcriptTextLive: string;
translateText: string; translateText: string;
accumulatedText: string;
title: string; title: string;
topics: Topic[]; topics: Topic[];
finalSummary: FinalSummary; finalSummary: FinalSummary;
@@ -17,7 +18,7 @@ export type UseWebSockets = {
}; };
export const useWebSockets = (transcriptId: string | null): UseWebSockets => { export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
const [transcriptText, setTranscriptText] = useState<string>(""); const [transcriptTextLive, setTranscriptTextLive] = useState<string>("");
const [translateText, setTranslateText] = useState<string>(""); const [translateText, setTranslateText] = useState<string>("");
const [title, setTitle] = useState<string>(""); const [title, setTitle] = useState<string>("");
const [textQueue, setTextQueue] = useState<string[]>([]); const [textQueue, setTextQueue] = useState<string[]>([]);
@@ -29,12 +30,14 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
const [finalSummary, setFinalSummary] = useState<FinalSummary>({ const [finalSummary, setFinalSummary] = useState<FinalSummary>({
summary: "", summary: "",
}); });
const [status, setStatus] = useState<Status>({ value: "initial" }); const [status, setStatus] = useState<Status>({ value: "" });
const { setError } = useError(); const { setError } = useError();
const { websocket_url } = useContext(DomainContext); const { websocket_url } = useContext(DomainContext);
const api = useApi(); const api = useApi();
const [accumulatedText, setAccumulatedText] = useState<string>("");
useEffect(() => { useEffect(() => {
if (isProcessing || textQueue.length === 0) { if (isProcessing || textQueue.length === 0) {
return; return;
@@ -42,13 +45,12 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
setIsProcessing(true); setIsProcessing(true);
const text = textQueue[0]; const text = textQueue[0];
setTranscriptText(text); setTranscriptTextLive(text);
setTranslateText(translationQueue[0]); setTranslateText(translationQueue[0]);
const WPM_READING = 200 + textQueue.length * 10; // words per minute to read const WPM_READING = 200 + textQueue.length * 10; // words per minute to read
const wordCount = text.split(/\s+/).length; const wordCount = text.split(/\s+/).length;
const delay = (wordCount / WPM_READING) * 60 * 1000; const delay = (wordCount / WPM_READING) * 60 * 1000;
console.log(`displaying "${text}" for ${delay}ms`);
setTimeout(() => { setTimeout(() => {
setIsProcessing(false); setIsProcessing(false);
setTextQueue((prevQueue) => prevQueue.slice(1)); setTextQueue((prevQueue) => prevQueue.slice(1));
@@ -92,7 +94,7 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
}, },
]; ];
setTranscriptText("Lorem Ipsum"); setTranscriptTextLive("Lorem Ipsum");
setTopics([ setTopics([
{ {
id: "1", id: "1",
@@ -190,9 +192,13 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
setFinalSummary({ summary: "This is the final summary" }); setFinalSummary({ summary: "This is the final summary" });
} }
if (e.key === "z" && process.env.NEXT_PUBLIC_ENV === "development") { if (e.key === "z" && process.env.NEXT_PUBLIC_ENV === "development") {
setTranscriptText( setTranscriptTextLive(
"This text is in English, and it is a pretty long sentence to test the limits", "This text is in English, and it is a pretty long sentence to test the limits",
); );
setAccumulatedText(
"This text is in English, and it is a pretty long sentence to test the limits. This text is in English, and it is a pretty long sentence to test the limits",
);
setStatus({ value: "recording" });
setTopics([ setTopics([
{ {
id: "1", id: "1",
@@ -333,6 +339,8 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
console.debug("TRANSCRIPT event:", newText); console.debug("TRANSCRIPT event:", newText);
setTextQueue((prevQueue) => [...prevQueue, newText]); setTextQueue((prevQueue) => [...prevQueue, newText]);
setTranslationQueue((prevQueue) => [...prevQueue, newTranslation]); setTranslationQueue((prevQueue) => [...prevQueue, newTranslation]);
setAccumulatedText((prevText) => prevText + " " + newText);
break; break;
case "TOPIC": case "TOPIC":
@@ -345,6 +353,10 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
prevTopics[index] = topic; prevTopics[index] = topic;
return prevTopics; return prevTopics;
} }
setAccumulatedText((prevText) =>
prevText.slice(topic.transcript.length),
);
return [...prevTopics, topic]; return [...prevTopics, topic];
}); });
console.debug("TOPIC event:", message.data); console.debug("TOPIC event:", message.data);
@@ -419,18 +431,18 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
break; break;
case 1005: // Closure by client FF case 1005: // Closure by client FF
break; break;
case 1001: // Navigate away
break;
default: default:
setError( setError(
new Error(`WebSocket closed unexpectedly with code: ${event.code}`), new Error(`WebSocket closed unexpectedly with code: ${event.code}`),
"Disconnected", "Disconnected from the server. Please refresh the page.",
); );
console.log( console.log(
"Socket is closed. Reconnect will be attempted in 1 second.", "Socket is closed. Reconnect will be attempted in 1 second.",
event.reason, event.reason,
); );
setTimeout(function () { // todo handle reconnect with socket.io
ws = new WebSocket(url);
}, 1000);
} }
}; };
@@ -440,8 +452,9 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
}, [transcriptId, !api]); }, [transcriptId, !api]);
return { return {
transcriptText, transcriptTextLive,
translateText, translateText,
accumulatedText,
topics, topics,
finalSummary, finalSummary,
title, title,

9
www/app/styles/icons/stopRecord.tsx Normal file
View File

@@ -0,0 +1,9 @@
import { Icon } from "@chakra-ui/react";
export default function StopRecordIcon(props) {
return (
<Icon viewBox="0 0 20 20" {...props}>
<rect width="20" height="20" rx="1" fill="currentColor" />
</Icon>
);
}