diff --git a/www/app/lib/CustomRecordPlugin.js b/www/app/lib/custom-plugins/record.ts similarity index 51% rename from www/app/lib/CustomRecordPlugin.js rename to www/app/lib/custom-plugins/record.ts index 7e29ea7c..2d42e80f 100644 --- a/www/app/lib/CustomRecordPlugin.js +++ b/www/app/lib/custom-plugins/record.ts @@ -1,7 +1,21 @@ -// Override the startRecording method so we can pass the desired stream -// Checkout: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts +// Source code: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts +/** + * Record audio from the microphone, render a waveform and download the audio. + */ -import RecordPlugin from "wavesurfer.js/dist/plugins/record"; +import BasePlugin, { + type BasePluginEvents, +} from "wavesurfer.js/dist/base-plugin"; + +export type RecordPluginOptions = { + mimeType?: MediaRecorderOptions["mimeType"]; + audioBitsPerSecond?: MediaRecorderOptions["audioBitsPerSecond"]; +}; + +export type RecordPluginEvents = BasePluginEvents & { + startRecording: []; + stopRecording: []; +}; const MIME_TYPES = [ "audio/webm", @@ -13,11 +27,44 @@ const MIME_TYPES = [ const findSupportedMimeType = () => MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType)); -class CustomRecordPlugin extends RecordPlugin { - static create(options) { - return new CustomRecordPlugin(options || {}); +class RecordPlugin extends BasePlugin { + private mediaRecorder: MediaRecorder | null = null; + private recordedUrl = ""; + private savedCursorWidth = 1; + private savedInteractive = true; + + public static create(options?: RecordPluginOptions) { + return new RecordPlugin(options || {}); } - render(stream) { + + private preventInteraction() { + if (this.wavesurfer) { + this.savedCursorWidth = this.wavesurfer.options.cursorWidth || 1; + this.savedInteractive = this.wavesurfer.options.interact || true; + this.wavesurfer.options.cursorWidth = 0; + this.wavesurfer.options.interact = false; + } + } + + private restoreInteraction() { + if (this.wavesurfer) { + this.wavesurfer.options.cursorWidth = this.savedCursorWidth; + this.wavesurfer.options.interact = this.savedInteractive; + } + } + + onInit() { + this.preventInteraction(); + } + + private loadBlob(data: Blob[], type: string) { + const blob = new Blob(data, { type }); + this.recordedUrl = URL.createObjectURL(blob); + this.restoreInteraction(); + this.wavesurfer?.load(this.recordedUrl); + } + + render(stream: MediaStream): () => void { if (!this.wavesurfer) return () => undefined; const container = this.wavesurfer.getWrapper(); @@ -36,11 +83,12 @@ class CustomRecordPlugin extends RecordPlugin { const bufferLength = analyser.frequencyBinCount; const dataArray = new Uint8Array(bufferLength); - let animationId, previousTimeStamp; + let animationId: number, previousTimeStamp: number; + const DATA_SIZE = 128.0; const BUFFER_SIZE = 2 ** 8; - const dataBuffer = new Array(BUFFER_SIZE).fill(canvas.height); + const dataBuffer = new Array(BUFFER_SIZE).fill(DATA_SIZE); - const drawWaveform = (timeStamp) => { + const drawWaveform = (timeStamp: number) => { if (!canvasCtx) return; analyser.getByteTimeDomainData(dataArray); @@ -64,9 +112,9 @@ class CustomRecordPlugin extends RecordPlugin { let x = 0; for (let i = 0; i < dataBuffer.length; i++) { - const valueNormalized = dataBuffer[i] / canvas.height; - const y = (valueNormalized * canvas.height) / 2; - const sliceHeight = canvas.height + 1 - y * 2; + const y = (canvas.height * dataBuffer[i]) / (2 * DATA_SIZE); + const sliceHeight = + ((1 - canvas.height) * dataBuffer[i]) / DATA_SIZE + canvas.height; canvasCtx.fillRect(x, y, (sliceWidth * 2) / 3, sliceHeight); x += sliceWidth; @@ -75,7 +123,7 @@ class CustomRecordPlugin extends RecordPlugin { animationId = requestAnimationFrame(drawWaveform); }; - drawWaveform(); + drawWaveform(0); return () => { if (animationId) { @@ -94,7 +142,17 @@ class CustomRecordPlugin extends RecordPlugin { canvas?.remove(); }; } - startRecording(stream) { + + private cleanUp() { + this.stopRecording(); + this.wavesurfer?.empty(); + if (this.recordedUrl) { + URL.revokeObjectURL(this.recordedUrl); + this.recordedUrl = ""; + } + } + + public async startRecording(stream: MediaStream) { this.preventInteraction(); this.cleanUp(); @@ -103,7 +161,7 @@ class CustomRecordPlugin extends RecordPlugin { mimeType: this.options.mimeType || findSupportedMimeType(), audioBitsPerSecond: this.options.audioBitsPerSecond, }); - const recordedChunks = []; + const recordedChunks: Blob[] = []; mediaRecorder.addEventListener("dataavailable", (event) => { if (event.data.size > 0) { @@ -123,6 +181,25 @@ class CustomRecordPlugin extends RecordPlugin { this.mediaRecorder = mediaRecorder; } + + public isRecording(): boolean { + return this.mediaRecorder?.state === "recording"; + } + + public stopRecording() { + if (this.isRecording()) { + this.mediaRecorder?.stop(); + } + } + + public getRecordedUrl(): string { + return this.recordedUrl; + } + + public destroy() { + super.destroy(); + this.cleanUp(); + } } -export default CustomRecordPlugin; +export default RecordPlugin; diff --git a/www/app/lib/custom-plugins/regions.ts b/www/app/lib/custom-plugins/regions.ts new file mode 100644 index 00000000..dff05f3b --- /dev/null +++ b/www/app/lib/custom-plugins/regions.ts @@ -0,0 +1,18 @@ +// Source code: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/regions.ts + +import RegionsPlugin, { + RegionsPluginOptions, +} from "wavesurfer.js/dist/plugins/regions"; + +class CustomRegionsPlugin extends RegionsPlugin { + public static create(options?: RegionsPluginOptions) { + return new CustomRegionsPlugin(options); + } + + constructor(options?: RegionsPluginOptions) { + super(options); + this["avoidOverlapping"] = () => {}; + } +} + +export default CustomRegionsPlugin; diff --git a/www/app/transcripts/dashboard.tsx b/www/app/transcripts/dashboard.tsx index 9eb05536..1f0c28e1 100644 --- a/www/app/transcripts/dashboard.tsx +++ b/www/app/transcripts/dashboard.tsx @@ -16,6 +16,10 @@ type DashboardProps = { finalSummary: FinalSummaryType; topics: Topic[]; disconnected: boolean; + useActiveTopic: [ + Topic | null, + React.Dispatch>, + ]; }; export function Dashboard({ @@ -23,8 +27,9 @@ export function Dashboard({ finalSummary, topics, disconnected, + useActiveTopic, }: DashboardProps) { - const [openIndex, setOpenIndex] = useState(null); + const [activeTopic, setActiveTopic] = useActiveTopic; const [autoscrollEnabled, setAutoscrollEnabled] = useState(true); useEffect(() => { @@ -51,7 +56,7 @@ export function Dashboard({ return ( <> -
+

Meeting Notes

@@ -75,18 +80,24 @@ export function Dashboard({
setOpenIndex(openIndex === index ? null : index)} + onClick={() => + setActiveTopic(activeTopic?.id == item.id ? null : item) + } >
{formatTime(item.timestamp)}
{item.title}
- {openIndex === index && ( + {activeTopic?.id == item.id && (
{item.transcript}
diff --git a/www/app/transcripts/new/page.tsx b/www/app/transcripts/new/page.tsx index 37a17066..7e065093 100644 --- a/www/app/transcripts/new/page.tsx +++ b/www/app/transcripts/new/page.tsx @@ -6,11 +6,13 @@ import useWebRTC from "../useWebRTC"; import useTranscript from "../useTranscript"; import { useWebSockets } from "../useWebSockets"; import "../../styles/button.css"; +import { Topic } from "../webSocketTypes"; import getApi from "../../lib/getApi"; const App = () => { const [stream, setStream] = useState(null); const [disconnected, setDisconnected] = useState(false); + const useActiveTopic = useState(null); useEffect(() => { if (process.env.NEXT_PUBLIC_ENV === "development") { @@ -35,6 +37,8 @@ const App = () => { webRTC?.peer?.send(JSON.stringify({ cmd: "STOP" })); setStream(null); }} + topics={webSockets.topics} + useActiveTopic={useActiveTopic} />
@@ -44,6 +48,7 @@ const App = () => { finalSummary={webSockets.finalSummary} topics={webSockets.topics} disconnected={disconnected} + useActiveTopic={useActiveTopic} /> ); diff --git a/www/app/transcripts/recorder.js b/www/app/transcripts/recorder.tsx similarity index 56% rename from www/app/transcripts/recorder.js rename to www/app/transcripts/recorder.tsx index 5eb06c82..6d0731ac 100644 --- a/www/app/transcripts/recorder.js +++ b/www/app/transcripts/recorder.tsx @@ -1,18 +1,22 @@ import React, { useRef, useEffect, useState } from "react"; import WaveSurfer from "wavesurfer.js"; +import RecordPlugin from "../lib/custom-plugins/record"; +import CustomRegionsPlugin from "../lib/custom-plugins/regions"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faDownload } from "@fortawesome/free-solid-svg-icons"; -import Dropdown from "react-dropdown"; +import Dropdown, { Option } from "react-dropdown"; import "react-dropdown/style.css"; -import CustomRecordPlugin from "../lib/CustomRecordPlugin"; import { formatTime } from "../lib/time"; -const AudioInputsDropdown = (props) => { - const [ddOptions, setDdOptions] = useState([]); +const AudioInputsDropdown: React.FC<{ + setDeviceId: React.Dispatch>; + disabled: boolean; +}> = (props) => { + const [ddOptions, setDdOptions] = useState>([]); useEffect(() => { const init = async () => { @@ -34,8 +38,8 @@ const AudioInputsDropdown = (props) => { init(); }, []); - const handleDropdownChange = (e) => { - props.setDeviceId(e.value); + const handleDropdownChange = (option: Option) => { + props.setDeviceId(option.value); }; return ( @@ -48,19 +52,27 @@ const AudioInputsDropdown = (props) => { ); }; -export default function Recorder(props) { - const waveformRef = useRef(); - const [wavesurfer, setWavesurfer] = useState(null); - const [record, setRecord] = useState(null); - const [isRecording, setIsRecording] = useState(false); - const [isPlaying, setIsPlaying] = useState(false); - const [deviceId, setDeviceId] = useState(null); - const [currentTime, setCurrentTime] = useState(0); - const [timeInterval, setTimeInterval] = useState(null); - const [duration, setDuration] = useState(0); +export default function Recorder(props: any) { + const waveformRef = useRef(null); + const [wavesurfer, setWavesurfer] = useState(null); + const [record, setRecord] = useState(null); + const [isRecording, setIsRecording] = useState(false); + const [isPlaying, setIsPlaying] = useState(false); + const [deviceId, setDeviceId] = useState(null); + const [currentTime, setCurrentTime] = useState(0); + const [timeInterval, setTimeInterval] = useState(null); + const [duration, setDuration] = useState(0); + const [waveRegions, setWaveRegions] = useState( + null, + ); + + const [activeTopic, setActiveTopic] = props.useActiveTopic; + + const topicsRef = useRef(props.topics); useEffect(() => { - document.getElementById("play-btn").disabled = true; + const playBtn = document.getElementById("play-btn"); + if (playBtn) playBtn.setAttribute("disabled", "true"); if (waveformRef.current) { const _wavesurfer = WaveSurfer.create({ @@ -71,6 +83,7 @@ export default function Recorder(props) { hideScrollbar: true, autoCenter: true, barWidth: 2, + height: 90, }); const wsWrapper = _wavesurfer.getWrapper(); wsWrapper.style.cursor = "pointer"; @@ -85,36 +98,97 @@ export default function Recorder(props) { }); _wavesurfer.on("timeupdate", setCurrentTime); - setRecord(_wavesurfer.registerPlugin(CustomRecordPlugin.create())); + setRecord(_wavesurfer.registerPlugin(RecordPlugin.create())); + setWaveRegions(_wavesurfer.registerPlugin(CustomRegionsPlugin.create())); + setWavesurfer(_wavesurfer); return () => { _wavesurfer.destroy(); setIsRecording(false); setIsPlaying(false); + setCurrentTime(0); }; } }, []); + useEffect(() => { + topicsRef.current = props.topics; + if (!isRecording) renderMarkers(); + }, [props.topics]); + + const renderMarkers = () => { + if (!waveRegions) return; + + waveRegions.clearRegions(); + for (let topic of topicsRef.current) { + const content = document.createElement("div"); + content.setAttribute( + "style", + ` + position: absolute; + border-left: solid 1px orange; + padding: 0 2px 0 5px; + font-size: 0.7rem; + width: 100px; + max-width: fit-content; + cursor: pointer; + background-color: white; + border-radius: 0 3px 3px 0; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + transition: width 100ms linear; + `, + ); + content.onmouseover = () => { + content.style.backgroundColor = "orange"; + content.style.zIndex = "999"; + content.style.width = "300px"; + }; + content.onmouseout = () => { + content.style.backgroundColor = "white"; + content.style.zIndex = "0"; + content.style.width = "100px"; + }; + content.textContent = topic.title; + + const region = waveRegions.addRegion({ + start: topic.timestamp, + content, + color: "f00", + drag: false, + }); + region.on("click", (e) => { + e.stopPropagation(); + setActiveTopic(topic); + wavesurfer?.setTime(region.start); + }); + } + }; + useEffect(() => { if (record) { return record.on("stopRecording", () => { const link = document.getElementById("download-recording"); - link.href = record.getRecordedUrl(); - link.download = "reflector-recording.webm"; + if (!link) return; + + link.setAttribute("href", record.getRecordedUrl()); + link.setAttribute("download", "reflector-recording.webm"); link.style.visibility = "visible"; + renderMarkers(); }); } }, [record]); useEffect(() => { if (isRecording) { - const interval = setInterval(() => { + const interval = window.setInterval(() => { setCurrentTime((prev) => prev + 1); }, 1000); setTimeInterval(interval); return () => clearInterval(interval); } else { - clearInterval(timeInterval); + clearInterval(timeInterval as number); setCurrentTime((prev) => { setDuration(prev); return 0; @@ -122,6 +196,12 @@ export default function Recorder(props) { } }, [isRecording]); + useEffect(() => { + if (activeTopic) { + wavesurfer?.setTime(activeTopic.timestamp); + } + }, [activeTopic]); + const handleRecClick = async () => { if (!record) return console.log("no record"); @@ -129,11 +209,12 @@ export default function Recorder(props) { props.onStop(); record.stopRecording(); setIsRecording(false); - document.getElementById("play-btn").disabled = false; + const playBtn = document.getElementById("play-btn"); + if (playBtn) playBtn.removeAttribute("disabled"); } else { const stream = await navigator.mediaDevices.getUserMedia({ audio: { - deviceId, + deviceId: deviceId as string, noiseSuppression: false, echoCancellation: false, }, @@ -141,6 +222,7 @@ export default function Recorder(props) { await record.startRecording(stream); props.setStream(stream); setIsRecording(true); + waveRegions?.clearRegions(); } }; diff --git a/www/tsconfig.json b/www/tsconfig.json index 9c9b16c2..0e1b89ae 100644 --- a/www/tsconfig.json +++ b/www/tsconfig.json @@ -18,7 +18,8 @@ "name": "next" } ], - "strictNullChecks": true + "strictNullChecks": true, + "downlevelIteration": true }, "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"]