diff --git a/server/README.md b/server/README.md index b0ca34b7..b01c9253 100644 --- a/server/README.md +++ b/server/README.md @@ -225,7 +225,7 @@ From the reflector root folder, run `python3 whisjax_realtime.py` -The transcription text should be written to `real_time_transcription_.txt`. +The transcription text should be written to `real_time_transcription_.txt` NEXT STEPS: diff --git a/www/README.md b/www/README.md index c92df561..03764cd4 100644 --- a/www/README.md +++ b/www/README.md @@ -78,7 +78,7 @@ This data is then returned from the `useWebRTC` hook and can be used in your com To generate the TypeScript files from the openapi.json file, make sure the python server is running, then run: -``` +```bash yarn openapi ``` diff --git a/www/app/lib/custom-plugins/regions.ts b/www/app/lib/custom-plugins/regions.ts new file mode 100644 index 00000000..dff05f3b --- /dev/null +++ b/www/app/lib/custom-plugins/regions.ts @@ -0,0 +1,18 @@ +// Source code: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/regions.ts + +import RegionsPlugin, { + RegionsPluginOptions, +} from "wavesurfer.js/dist/plugins/regions"; + +class CustomRegionsPlugin extends RegionsPlugin { + public static create(options?: RegionsPluginOptions) { + return new CustomRegionsPlugin(options); + } + + constructor(options?: RegionsPluginOptions) { + super(options); + this["avoidOverlapping"] = () => {}; + } +} + +export default CustomRegionsPlugin; diff --git a/www/app/transcripts/dashboard.tsx b/www/app/transcripts/dashboard.tsx index 9eb05536..1f0c28e1 100644 --- a/www/app/transcripts/dashboard.tsx +++ b/www/app/transcripts/dashboard.tsx @@ -16,6 +16,10 @@ type DashboardProps = { finalSummary: FinalSummaryType; topics: Topic[]; disconnected: boolean; + useActiveTopic: [ + Topic | null, + React.Dispatch>, + ]; }; export function Dashboard({ @@ -23,8 +27,9 @@ export function Dashboard({ finalSummary, topics, disconnected, + useActiveTopic, }: DashboardProps) { - const [openIndex, setOpenIndex] = useState(null); + const [activeTopic, setActiveTopic] = useActiveTopic; const [autoscrollEnabled, setAutoscrollEnabled] = useState(true); useEffect(() => { @@ -51,7 +56,7 @@ export function Dashboard({ return ( <> -
+

Meeting Notes

@@ -75,18 +80,24 @@ export function Dashboard({
setOpenIndex(openIndex === index ? null : index)} + onClick={() => + setActiveTopic(activeTopic?.id == item.id ? null : item) + } >
{formatTime(item.timestamp)}
{item.title}
- {openIndex === index && ( + {activeTopic?.id == item.id && (
{item.transcript}
diff --git a/www/app/transcripts/new/page.tsx b/www/app/transcripts/new/page.tsx index 24134f57..06908e89 100644 --- a/www/app/transcripts/new/page.tsx +++ b/www/app/transcripts/new/page.tsx @@ -7,11 +7,13 @@ import useTranscript from "../useTranscript"; import { useWebSockets } from "../useWebSockets"; import useAudioDevice from "../useAudioDevice"; import "../../styles/button.css"; +import { Topic } from "../webSocketTypes"; import getApi from "../../lib/getApi"; const App = () => { const [stream, setStream] = useState(null); const [disconnected, setDisconnected] = useState(false); + const useActiveTopic = useState(null); useEffect(() => { if (process.env.NEXT_PUBLIC_ENV === "development") { @@ -48,6 +50,7 @@ const App = () => { }} getAudioStream={getAudioStream} audioDevices={audioDevices} + topics={webSockets.topics} /> { finalSummary={webSockets.finalSummary} topics={webSockets.topics} disconnected={disconnected} + useActiveTopic={useActiveTopic} /> ) : ( diff --git a/www/app/transcripts/recorder.tsx b/www/app/transcripts/recorder.tsx index 633fe1cd..0e3c01d7 100644 --- a/www/app/transcripts/recorder.tsx +++ b/www/app/transcripts/recorder.tsx @@ -2,6 +2,7 @@ import React, { useRef, useEffect, useState } from "react"; import WaveSurfer from "wavesurfer.js"; import RecordPlugin from "../lib/custom-plugins/record"; +import CustomRegionsPlugin from "../lib/custom-plugins/regions"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faDownload } from "@fortawesome/free-solid-svg-icons"; @@ -11,11 +12,11 @@ import "react-dropdown/style.css"; import { formatTime } from "../lib/time"; -const AudioInputsDropdown = (props: { +const AudioInputsDropdown: React.FC<{ audioDevices: Option[]; setDeviceId: React.Dispatch>; disabled: boolean; -}) => { +}> = (props) => { const [ddOptions, setDdOptions] = useState([]); useEffect(() => { @@ -39,7 +40,7 @@ const AudioInputsDropdown = (props: { ); }; -export default function Recorder(props) { +export default function Recorder(props: any) { const waveformRef = useRef(null); const [wavesurfer, setWavesurfer] = useState(null); const [record, setRecord] = useState(null); @@ -49,6 +50,13 @@ export default function Recorder(props) { const [currentTime, setCurrentTime] = useState(0); const [timeInterval, setTimeInterval] = useState(null); const [duration, setDuration] = useState(0); + const [waveRegions, setWaveRegions] = useState( + null, + ); + + const [activeTopic, setActiveTopic] = props.useActiveTopic; + + const topicsRef = useRef(props.topics); useEffect(() => { const playBtn = document.getElementById("play-btn"); @@ -63,6 +71,7 @@ export default function Recorder(props) { hideScrollbar: true, autoCenter: true, barWidth: 2, + height: 90, }); const wsWrapper = _wavesurfer.getWrapper(); wsWrapper.style.cursor = "pointer"; @@ -78,15 +87,73 @@ export default function Recorder(props) { _wavesurfer.on("timeupdate", setCurrentTime); setRecord(_wavesurfer.registerPlugin(RecordPlugin.create())); + setWaveRegions(_wavesurfer.registerPlugin(CustomRegionsPlugin.create())); + setWavesurfer(_wavesurfer); return () => { _wavesurfer.destroy(); setIsRecording(false); setIsPlaying(false); + setCurrentTime(0); }; } }, []); + useEffect(() => { + topicsRef.current = props.topics; + if (!isRecording) renderMarkers(); + }, [props.topics]); + + const renderMarkers = () => { + if (!waveRegions) return; + + waveRegions.clearRegions(); + for (let topic of topicsRef.current) { + const content = document.createElement("div"); + content.setAttribute( + "style", + ` + position: absolute; + border-left: solid 1px orange; + padding: 0 2px 0 5px; + font-size: 0.7rem; + width: 100px; + max-width: fit-content; + cursor: pointer; + background-color: white; + border-radius: 0 3px 3px 0; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + transition: width 100ms linear; + `, + ); + content.onmouseover = () => { + content.style.backgroundColor = "orange"; + content.style.zIndex = "999"; + content.style.width = "300px"; + }; + content.onmouseout = () => { + content.style.backgroundColor = "white"; + content.style.zIndex = "0"; + content.style.width = "100px"; + }; + content.textContent = topic.title; + + const region = waveRegions.addRegion({ + start: topic.timestamp, + content, + color: "f00", + drag: false, + }); + region.on("click", (e) => { + e.stopPropagation(); + setActiveTopic(topic); + wavesurfer?.setTime(region.start); + }); + } + }; + useEffect(() => { if (record) { return record.on("stopRecording", () => { @@ -96,6 +163,7 @@ export default function Recorder(props) { link.setAttribute("href", record.getRecordedUrl()); link.setAttribute("download", "reflector-recording.webm"); link.style.visibility = "visible"; + renderMarkers(); }); } }, [record]); @@ -116,6 +184,12 @@ export default function Recorder(props) { } }, [isRecording]); + useEffect(() => { + if (activeTopic) { + wavesurfer?.setTime(activeTopic.timestamp); + } + }, [activeTopic]); + const handleRecClick = async () => { if (!record) return console.log("no record"); @@ -128,6 +202,7 @@ export default function Recorder(props) { } else { const stream = await props.getAudioStream(deviceId); props.setStream(stream); + waveRegions?.clearRegions(); if (stream) { await record.startRecording(stream); setIsRecording(true);