From 9c9428c456f99addd467ecd11b4de6e3103ba3f8 Mon Sep 17 00:00:00 2001 From: Jose B Date: Mon, 14 Aug 2023 03:02:56 -0500 Subject: [PATCH 01/15] reduce waveform height + fix weird math --- www/app/components/CustomRecordPlugin.js | 9 +++++---- www/app/components/dashboard.js | 2 +- www/app/components/record.js | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/www/app/components/CustomRecordPlugin.js b/www/app/components/CustomRecordPlugin.js index 7e29ea7c..ab99170f 100644 --- a/www/app/components/CustomRecordPlugin.js +++ b/www/app/components/CustomRecordPlugin.js @@ -37,8 +37,9 @@ class CustomRecordPlugin extends RecordPlugin { const dataArray = new Uint8Array(bufferLength); let animationId, previousTimeStamp; + const DATA_SIZE = 128.0; const BUFFER_SIZE = 2 ** 8; - const dataBuffer = new Array(BUFFER_SIZE).fill(canvas.height); + const dataBuffer = new Array(BUFFER_SIZE).fill(DATA_SIZE); const drawWaveform = (timeStamp) => { if (!canvasCtx) return; @@ -64,9 +65,9 @@ class CustomRecordPlugin extends RecordPlugin { let x = 0; for (let i = 0; i < dataBuffer.length; i++) { - const valueNormalized = dataBuffer[i] / canvas.height; - const y = (valueNormalized * canvas.height) / 2; - const sliceHeight = canvas.height + 1 - y * 2; + const y = (canvas.height * dataBuffer[i]) / (2 * DATA_SIZE); + const sliceHeight = + ((1 - canvas.height) * dataBuffer[i]) / DATA_SIZE + canvas.height; canvasCtx.fillRect(x, y, (sliceWidth * 2) / 3, sliceHeight); x += sliceWidth; diff --git a/www/app/components/dashboard.js b/www/app/components/dashboard.js index 94bae58c..6f4f4b68 100644 --- a/www/app/components/dashboard.js +++ b/www/app/components/dashboard.js @@ -48,7 +48,7 @@ export function Dashboard({ return ( <> -
+

Meeting Notes

diff --git a/www/app/components/record.js b/www/app/components/record.js index 445f7e50..13f45257 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -71,6 +71,7 @@ export default function Recorder(props) { hideScrollbar: true, autoCenter: true, barWidth: 2, + height: 90, }); const wsWrapper = _wavesurfer.getWrapper(); wsWrapper.style.cursor = "pointer"; From 5ed203547feacbc0ca292e0d92d108c33efeaf42 Mon Sep 17 00:00:00 2001 From: Jose B Date: Mon, 14 Aug 2023 12:14:33 -0500 Subject: [PATCH 02/15] add markers + sync w/ topics --- www/app/components/dashboard.js | 17 +++++++++++++---- www/app/components/record.js | 26 ++++++++++++++++++++++++++ www/app/page.js | 4 ++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/www/app/components/dashboard.js b/www/app/components/dashboard.js index 6f4f4b68..d7d5dde9 100644 --- a/www/app/components/dashboard.js +++ b/www/app/components/dashboard.js @@ -11,8 +11,9 @@ export function Dashboard({ finalSummary, topics, disconnected, + useActiveTopic, }) { - const [openIndex, setOpenIndex] = useState(null); + const [activeTopic, setActiveTopic] = useActiveTopic; const [autoscrollEnabled, setAutoscrollEnabled] = useState(true); useEffect(() => { @@ -76,18 +77,26 @@ export function Dashboard({
setOpenIndex(openIndex === index ? null : index)} + onClick={() => + setActiveTopic( + activeTopic == item.timestamp ? null : item.timestamp, + ) + } >
{formatTime(item.timestamp)}
{item.title}
- {openIndex === index && ( + {activeTopic == item.timestamp && (
{item.transcript}
diff --git a/www/app/components/record.js b/www/app/components/record.js index 13f45257..0885b802 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -1,6 +1,7 @@ import React, { useRef, useEffect, useState } from "react"; import WaveSurfer from "wavesurfer.js"; +import RegionsPlugin from "wavesurfer.js/dist/plugins/regions"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faDownload } from "@fortawesome/free-solid-svg-icons"; @@ -58,6 +59,9 @@ export default function Recorder(props) { const [currentTime, setCurrentTime] = useState(0); const [timeInterval, setTimeInterval] = useState(null); const [duration, setDuration] = useState(0); + const [waveRegions, setWaveRegions] = useState(null); + + const [activeTopic, setActiveTopic] = props.useActiveTopic; useEffect(() => { document.getElementById("play-btn").disabled = true; @@ -87,6 +91,8 @@ export default function Recorder(props) { _wavesurfer.on("timeupdate", setCurrentTime); setRecord(_wavesurfer.registerPlugin(CustomRecordPlugin.create())); + setWaveRegions(_wavesurfer.registerPlugin(RegionsPlugin.create())); + setWavesurfer(_wavesurfer); return () => { _wavesurfer.destroy(); @@ -103,6 +109,20 @@ export default function Recorder(props) { link.href = record.getRecordedUrl(); link.download = "reflector-recording.webm"; link.style.visibility = "visible"; + + for (let topic of props.topics) { + const region = waveRegions.addRegion({ + start: topic.timestamp, + content: topic.title.slice(0, 7) + "...", + color: "f00", + drag: false, + }); + region.on("click", (e) => { + e.stopPropagation(); + setActiveTopic(region.start); + wavesurfer.setTime(region.start); + }); + } }); } }, [record]); @@ -123,6 +143,12 @@ export default function Recorder(props) { } }, [isRecording]); + useEffect(() => { + if (activeTopic) { + wavesurfer.setTime(activeTopic); + } + }, [activeTopic]); + const handleRecClick = async () => { if (!record) return console.log("no record"); diff --git a/www/app/page.js b/www/app/page.js index 3ebf8e32..c3bcbf3a 100644 --- a/www/app/page.js +++ b/www/app/page.js @@ -10,6 +10,7 @@ import "../public/button.css"; const App = () => { const [stream, setStream] = useState(null); const [disconnected, setDisconnected] = useState(false); + const useActiveTopic = useState(null); useEffect(() => { if (process.env.NEXT_PUBLIC_ENV === "development") { @@ -38,6 +39,8 @@ const App = () => { webRTC?.peer?.send(JSON.stringify({ cmd: "STOP" })); setStream(null); }} + topics={webSockets.topics} + useActiveTopic={useActiveTopic} /> { topics={webSockets.topics} stream={stream} disconnected={disconnected} + useActiveTopic={useActiveTopic} />
); From d0109a7f751a30fbf493f83dd7b51d2430cf7d4c Mon Sep 17 00:00:00 2001 From: Jose B Date: Mon, 14 Aug 2023 13:57:32 -0500 Subject: [PATCH 03/15] =?UTF-8?q?style=20it=20=F0=9F=A7=91=E2=80=8D?= =?UTF-8?q?=F0=9F=8E=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- www/app/components/record.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/www/app/components/record.js b/www/app/components/record.js index 0885b802..3b178125 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -111,9 +111,28 @@ export default function Recorder(props) { link.style.visibility = "visible"; for (let topic of props.topics) { + const content = document.createElement("div"); + content.style = ` + border-left: solid 1px orange; + padding: 0 2px; + font-size: 0.7rem; + max-width: 100px; + width: max-content; + cursor: pointer; + background-color: white; + border-radius: 0 3px 3px 0; + `; + content.onmouseover = () => + (content.style.backgroundColor = "orange"); + content.onmouseout = () => (content.style.backgroundColor = "white"); + content.textContent = + topic.title.length >= 20 + ? topic.title.slice(0, 17) + "..." + : topic.title; + const region = waveRegions.addRegion({ start: topic.timestamp, - content: topic.title.slice(0, 7) + "...", + content, color: "f00", drag: false, }); From 4ef844fe99dfb8361a46e606d12768150d89aaa3 Mon Sep 17 00:00:00 2001 From: Jose B Date: Mon, 14 Aug 2023 18:23:24 -0500 Subject: [PATCH 04/15] use topic object instead timestamp --- www/app/components/dashboard.js | 8 +++----- www/app/components/record.js | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/www/app/components/dashboard.js b/www/app/components/dashboard.js index d7d5dde9..79e15efb 100644 --- a/www/app/components/dashboard.js +++ b/www/app/components/dashboard.js @@ -78,9 +78,7 @@ export function Dashboard({
- setActiveTopic( - activeTopic == item.timestamp ? null : item.timestamp, - ) + setActiveTopic(activeTopic?.id == item.id ? null : item) } >
{formatTime(item.timestamp)}
@@ -89,14 +87,14 @@ export function Dashboard({
- {activeTopic == item.timestamp && ( + {activeTopic?.id == item.id && (
{item.transcript}
diff --git a/www/app/components/record.js b/www/app/components/record.js index 3b178125..c77d3838 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -138,7 +138,7 @@ export default function Recorder(props) { }); region.on("click", (e) => { e.stopPropagation(); - setActiveTopic(region.start); + setActiveTopic(topic); wavesurfer.setTime(region.start); }); } @@ -164,7 +164,7 @@ export default function Recorder(props) { useEffect(() => { if (activeTopic) { - wavesurfer.setTime(activeTopic); + wavesurfer.setTime(activeTopic.timestamp); } }, [activeTopic]); From 6d7c323203664a1ade07301ac4180d3ef2cac27b Mon Sep 17 00:00:00 2001 From: Jose B Date: Mon, 14 Aug 2023 18:54:12 -0500 Subject: [PATCH 05/15] fix stale props --- www/app/components/record.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/www/app/components/record.js b/www/app/components/record.js index c77d3838..44de590e 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -63,6 +63,8 @@ export default function Recorder(props) { const [activeTopic, setActiveTopic] = props.useActiveTopic; + const topicsRef = useRef(props.topics); + useEffect(() => { document.getElementById("play-btn").disabled = true; @@ -102,6 +104,10 @@ export default function Recorder(props) { } }, []); + useEffect(() => { + topicsRef.current = props.topics; + }, [props.topics]); + useEffect(() => { if (record) { return record.on("stopRecording", () => { @@ -110,7 +116,7 @@ export default function Recorder(props) { link.download = "reflector-recording.webm"; link.style.visibility = "visible"; - for (let topic of props.topics) { + for (let topic of topicsRef.current) { const content = document.createElement("div"); content.style = ` border-left: solid 1px orange; From d471cdd0a59ed1778728446b484c70cdfaae4797 Mon Sep 17 00:00:00 2001 From: Jose B Date: Mon, 14 Aug 2023 22:02:52 -0500 Subject: [PATCH 06/15] fix markers not rendering --- www/app/components/record.js | 72 +++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/www/app/components/record.js b/www/app/components/record.js index 44de590e..c47a2edd 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -106,8 +106,46 @@ export default function Recorder(props) { useEffect(() => { topicsRef.current = props.topics; + if (!isRecording) renderMarkers(); }, [props.topics]); + const renderMarkers = () => { + if (!waveRegions) return; + + waveRegions.clearRegions(); + for (let topic of topicsRef.current) { + const content = document.createElement("div"); + content.style = ` + border-left: solid 1px orange; + padding: 0 2px; + font-size: 0.7rem; + max-width: 100px; + width: max-content; + cursor: pointer; + background-color: white; + border-radius: 0 3px 3px 0; + `; + content.onmouseover = () => (content.style.backgroundColor = "orange"); + content.onmouseout = () => (content.style.backgroundColor = "white"); + content.textContent = + topic.title.length >= 20 + ? topic.title.slice(0, 17) + "..." + : topic.title; + + const region = waveRegions.addRegion({ + start: topic.timestamp, + content, + color: "f00", + drag: false, + }); + region.on("click", (e) => { + e.stopPropagation(); + setActiveTopic(topic); + wavesurfer.setTime(region.start); + }); + } + }; + useEffect(() => { if (record) { return record.on("stopRecording", () => { @@ -115,39 +153,7 @@ export default function Recorder(props) { link.href = record.getRecordedUrl(); link.download = "reflector-recording.webm"; link.style.visibility = "visible"; - - for (let topic of topicsRef.current) { - const content = document.createElement("div"); - content.style = ` - border-left: solid 1px orange; - padding: 0 2px; - font-size: 0.7rem; - max-width: 100px; - width: max-content; - cursor: pointer; - background-color: white; - border-radius: 0 3px 3px 0; - `; - content.onmouseover = () => - (content.style.backgroundColor = "orange"); - content.onmouseout = () => (content.style.backgroundColor = "white"); - content.textContent = - topic.title.length >= 20 - ? topic.title.slice(0, 17) + "..." - : topic.title; - - const region = waveRegions.addRegion({ - start: topic.timestamp, - content, - color: "f00", - drag: false, - }); - region.on("click", (e) => { - e.stopPropagation(); - setActiveTopic(topic); - wavesurfer.setTime(region.start); - }); - } + renderMarkers(); }); } }, [record]); From 6c99556221cbd65565da0cea84edea4862be4ed2 Mon Sep 17 00:00:00 2001 From: Jose B Date: Wed, 16 Aug 2023 17:16:19 -0500 Subject: [PATCH 07/15] =?UTF-8?q?re-arrange=20custom=20plugins,=20prevent?= =?UTF-8?q?=20overlapping,=20more=20=F0=9F=8E=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../record.js} | 0 www/app/transcripts/custom-plugins/regions.js | 12 +++++++ www/app/transcripts/new/page.js | 4 +++ www/app/transcripts/recorder.js | 33 ++++++++++++------- 4 files changed, 37 insertions(+), 12 deletions(-) rename www/app/transcripts/{CustomRecordPlugin.js => custom-plugins/record.js} (100%) create mode 100644 www/app/transcripts/custom-plugins/regions.js diff --git a/www/app/transcripts/CustomRecordPlugin.js b/www/app/transcripts/custom-plugins/record.js similarity index 100% rename from www/app/transcripts/CustomRecordPlugin.js rename to www/app/transcripts/custom-plugins/record.js diff --git a/www/app/transcripts/custom-plugins/regions.js b/www/app/transcripts/custom-plugins/regions.js new file mode 100644 index 00000000..528e99a3 --- /dev/null +++ b/www/app/transcripts/custom-plugins/regions.js @@ -0,0 +1,12 @@ +import RegionsPlugin from "wavesurfer.js/dist/plugins/regions"; + +class CustomRegionsPlugin extends RegionsPlugin { + static create(options) { + return new CustomRegionsPlugin(options); + } + avoidOverlapping(region) { + // Prevent overlapping regions + } +} + +export default CustomRegionsPlugin; diff --git a/www/app/transcripts/new/page.js b/www/app/transcripts/new/page.js index 60f4642b..d770fd16 100644 --- a/www/app/transcripts/new/page.js +++ b/www/app/transcripts/new/page.js @@ -10,6 +10,7 @@ import "../../styles/button.css"; const App = () => { const [stream, setStream] = useState(null); const [disconnected, setDisconnected] = useState(false); + const useActiveTopic = useState(null); useEffect(() => { if (process.env.NEXT_PUBLIC_ENV === "development") { @@ -38,6 +39,8 @@ const App = () => { webRTC?.peer?.send(JSON.stringify({ cmd: "STOP" })); setStream(null); }} + topics={webSockets.topics} + useActiveTopic={useActiveTopic} />
@@ -48,6 +51,7 @@ const App = () => { topics={webSockets.topics} stream={stream} disconnected={disconnected} + useActiveTopic={useActiveTopic} />
); diff --git a/www/app/transcripts/recorder.js b/www/app/transcripts/recorder.js index 661af26b..cf3e78fc 100644 --- a/www/app/transcripts/recorder.js +++ b/www/app/transcripts/recorder.js @@ -1,7 +1,8 @@ import React, { useRef, useEffect, useState } from "react"; import WaveSurfer from "wavesurfer.js"; -import RegionsPlugin from "wavesurfer.js/dist/plugins/regions"; +import CustomRecordPlugin from "./custom-plugins/record"; +import CustomRegionsPlugin from "./custom-plugins/regions"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faDownload } from "@fortawesome/free-solid-svg-icons"; @@ -9,7 +10,6 @@ import { faDownload } from "@fortawesome/free-solid-svg-icons"; import Dropdown from "react-dropdown"; import "react-dropdown/style.css"; -import CustomRecordPlugin from "./CustomRecordPlugin"; import { formatTime } from "../lib/time"; const AudioInputsDropdown = (props) => { @@ -93,7 +93,7 @@ export default function Recorder(props) { _wavesurfer.on("timeupdate", setCurrentTime); setRecord(_wavesurfer.registerPlugin(CustomRecordPlugin.create())); - setWaveRegions(_wavesurfer.registerPlugin(RegionsPlugin.create())); + setWaveRegions(_wavesurfer.registerPlugin(CustomRegionsPlugin.create())); setWavesurfer(_wavesurfer); return () => { @@ -116,21 +116,29 @@ export default function Recorder(props) { for (let topic of topicsRef.current) { const content = document.createElement("div"); content.style = ` + position: absolute; border-left: solid 1px orange; - padding: 0 2px; + padding: 0 2px 0 5px; font-size: 0.7rem; - max-width: 100px; - width: max-content; + width: 100px; cursor: pointer; background-color: white; border-radius: 0 3px 3px 0; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; `; - content.onmouseover = () => (content.style.backgroundColor = "orange"); - content.onmouseout = () => (content.style.backgroundColor = "white"); - content.textContent = - topic.title.length >= 20 - ? topic.title.slice(0, 17) + "..." - : topic.title; + content.onmouseover = () => { + content.style.backgroundColor = "orange"; + content.style.zIndex = 999; + content.style.width = "auto"; + }; + content.onmouseout = () => { + content.style.backgroundColor = "white"; + content.style.zIndex = 0; + content.style.width = "100px"; + }; + content.textContent = topic.title; const region = waveRegions.addRegion({ start: topic.timestamp, @@ -199,6 +207,7 @@ export default function Recorder(props) { await record.startRecording(stream); props.setStream(stream); setIsRecording(true); + waveRegions?.clearRegions(); } }; From 62cfa5b4460536a4c5ffd1f983a715ffbb5f76dc Mon Sep 17 00:00:00 2001 From: Jose B Date: Wed, 16 Aug 2023 20:53:53 -0500 Subject: [PATCH 08/15] add transition --- www/app/transcripts/recorder.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/www/app/transcripts/recorder.js b/www/app/transcripts/recorder.js index cf3e78fc..6016db8b 100644 --- a/www/app/transcripts/recorder.js +++ b/www/app/transcripts/recorder.js @@ -121,17 +121,19 @@ export default function Recorder(props) { padding: 0 2px 0 5px; font-size: 0.7rem; width: 100px; + max-width: fit-content; cursor: pointer; background-color: white; border-radius: 0 3px 3px 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; + transition: width 100ms linear; `; content.onmouseover = () => { content.style.backgroundColor = "orange"; content.style.zIndex = 999; - content.style.width = "auto"; + content.style.width = "300px"; }; content.onmouseout = () => { content.style.backgroundColor = "white"; From e390bda83ebf5a2b538cb306ca0c1feb44244a7e Mon Sep 17 00:00:00 2001 From: Jose B Date: Fri, 18 Aug 2023 14:12:48 -0500 Subject: [PATCH 09/15] initial TS implementation --- www/app/lib/custom-plugins/record.js | 129 ------------------ .../record.ts} | 3 +- www/app/lib/custom-plugins/regions.js | 12 -- www/app/lib/custom-plugins/regions.ts | 18 +++ www/app/transcripts/custom-plugins/record.js | 129 ------------------ www/app/transcripts/custom-plugins/regions.js | 12 -- 6 files changed, 19 insertions(+), 284 deletions(-) delete mode 100644 www/app/lib/custom-plugins/record.js rename www/app/lib/{CustomRecordPlugin.js => custom-plugins/record.ts} (95%) delete mode 100644 www/app/lib/custom-plugins/regions.js create mode 100644 www/app/lib/custom-plugins/regions.ts delete mode 100644 www/app/transcripts/custom-plugins/record.js delete mode 100644 www/app/transcripts/custom-plugins/regions.js diff --git a/www/app/lib/custom-plugins/record.js b/www/app/lib/custom-plugins/record.js deleted file mode 100644 index ab99170f..00000000 --- a/www/app/lib/custom-plugins/record.js +++ /dev/null @@ -1,129 +0,0 @@ -// Override the startRecording method so we can pass the desired stream -// Checkout: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts - -import RecordPlugin from "wavesurfer.js/dist/plugins/record"; - -const MIME_TYPES = [ - "audio/webm", - "audio/wav", - "audio/mpeg", - "audio/mp4", - "audio/mp3", -]; -const findSupportedMimeType = () => - MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType)); - -class CustomRecordPlugin extends RecordPlugin { - static create(options) { - return new CustomRecordPlugin(options || {}); - } - render(stream) { - if (!this.wavesurfer) return () => undefined; - - const container = this.wavesurfer.getWrapper(); - const canvas = document.createElement("canvas"); - canvas.width = container.clientWidth; - canvas.height = container.clientHeight; - canvas.style.zIndex = "10"; - container.appendChild(canvas); - - const canvasCtx = canvas.getContext("2d"); - const audioContext = new AudioContext(); - const source = audioContext.createMediaStreamSource(stream); - const analyser = audioContext.createAnalyser(); - analyser.fftSize = 2 ** 5; - source.connect(analyser); - const bufferLength = analyser.frequencyBinCount; - const dataArray = new Uint8Array(bufferLength); - - let animationId, previousTimeStamp; - const DATA_SIZE = 128.0; - const BUFFER_SIZE = 2 ** 8; - const dataBuffer = new Array(BUFFER_SIZE).fill(DATA_SIZE); - - const drawWaveform = (timeStamp) => { - if (!canvasCtx) return; - - analyser.getByteTimeDomainData(dataArray); - canvasCtx.clearRect(0, 0, canvas.width, canvas.height); - canvasCtx.fillStyle = "#cc3347"; - - if (previousTimeStamp === undefined) { - previousTimeStamp = timeStamp; - dataBuffer.push(Math.min(...dataArray)); - dataBuffer.splice(0, 1); - } - const elapsed = timeStamp - previousTimeStamp; - if (elapsed > 10) { - previousTimeStamp = timeStamp; - dataBuffer.push(Math.min(...dataArray)); - dataBuffer.splice(0, 1); - } - - // Drawing - const sliceWidth = canvas.width / dataBuffer.length; - let x = 0; - - for (let i = 0; i < dataBuffer.length; i++) { - const y = (canvas.height * dataBuffer[i]) / (2 * DATA_SIZE); - const sliceHeight = - ((1 - canvas.height) * dataBuffer[i]) / DATA_SIZE + canvas.height; - - canvasCtx.fillRect(x, y, (sliceWidth * 2) / 3, sliceHeight); - x += sliceWidth; - } - - animationId = requestAnimationFrame(drawWaveform); - }; - - drawWaveform(); - - return () => { - if (animationId) { - cancelAnimationFrame(animationId); - } - - if (source) { - source.disconnect(); - source.mediaStream.getTracks().forEach((track) => track.stop()); - } - - if (audioContext) { - audioContext.close(); - } - - canvas?.remove(); - }; - } - startRecording(stream) { - this.preventInteraction(); - this.cleanUp(); - - const onStop = this.render(stream); - const mediaRecorder = new MediaRecorder(stream, { - mimeType: this.options.mimeType || findSupportedMimeType(), - audioBitsPerSecond: this.options.audioBitsPerSecond, - }); - const recordedChunks = []; - - mediaRecorder.addEventListener("dataavailable", (event) => { - if (event.data.size > 0) { - recordedChunks.push(event.data); - } - }); - - mediaRecorder.addEventListener("stop", () => { - onStop(); - this.loadBlob(recordedChunks, mediaRecorder.mimeType); - this.emit("stopRecording"); - }); - - mediaRecorder.start(); - - this.emit("startRecording"); - - this.mediaRecorder = mediaRecorder; - } -} - -export default CustomRecordPlugin; diff --git a/www/app/lib/CustomRecordPlugin.js b/www/app/lib/custom-plugins/record.ts similarity index 95% rename from www/app/lib/CustomRecordPlugin.js rename to www/app/lib/custom-plugins/record.ts index ab99170f..aa85a812 100644 --- a/www/app/lib/CustomRecordPlugin.js +++ b/www/app/lib/custom-plugins/record.ts @@ -1,5 +1,4 @@ -// Override the startRecording method so we can pass the desired stream -// Checkout: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts +// Source code: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts import RecordPlugin from "wavesurfer.js/dist/plugins/record"; diff --git a/www/app/lib/custom-plugins/regions.js b/www/app/lib/custom-plugins/regions.js deleted file mode 100644 index 528e99a3..00000000 --- a/www/app/lib/custom-plugins/regions.js +++ /dev/null @@ -1,12 +0,0 @@ -import RegionsPlugin from "wavesurfer.js/dist/plugins/regions"; - -class CustomRegionsPlugin extends RegionsPlugin { - static create(options) { - return new CustomRegionsPlugin(options); - } - avoidOverlapping(region) { - // Prevent overlapping regions - } -} - -export default CustomRegionsPlugin; diff --git a/www/app/lib/custom-plugins/regions.ts b/www/app/lib/custom-plugins/regions.ts new file mode 100644 index 00000000..12fb7012 --- /dev/null +++ b/www/app/lib/custom-plugins/regions.ts @@ -0,0 +1,18 @@ +// Source code: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/regions.ts + +import RegionsPlugin, { + RegionsPluginOptions, +} from "wavesurfer.js/dist/plugins/regions"; + +class CustomRegionsPlugin extends RegionsPlugin { + public static create(options?: RegionsPluginOptions) { + return new RegionsPlugin(options); + } + + constructor(options?: RegionsPluginOptions) { + super(options); + this["avoidOverlapping"] = () => {}; + } +} + +export default CustomRegionsPlugin; diff --git a/www/app/transcripts/custom-plugins/record.js b/www/app/transcripts/custom-plugins/record.js deleted file mode 100644 index ab99170f..00000000 --- a/www/app/transcripts/custom-plugins/record.js +++ /dev/null @@ -1,129 +0,0 @@ -// Override the startRecording method so we can pass the desired stream -// Checkout: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts - -import RecordPlugin from "wavesurfer.js/dist/plugins/record"; - -const MIME_TYPES = [ - "audio/webm", - "audio/wav", - "audio/mpeg", - "audio/mp4", - "audio/mp3", -]; -const findSupportedMimeType = () => - MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType)); - -class CustomRecordPlugin extends RecordPlugin { - static create(options) { - return new CustomRecordPlugin(options || {}); - } - render(stream) { - if (!this.wavesurfer) return () => undefined; - - const container = this.wavesurfer.getWrapper(); - const canvas = document.createElement("canvas"); - canvas.width = container.clientWidth; - canvas.height = container.clientHeight; - canvas.style.zIndex = "10"; - container.appendChild(canvas); - - const canvasCtx = canvas.getContext("2d"); - const audioContext = new AudioContext(); - const source = audioContext.createMediaStreamSource(stream); - const analyser = audioContext.createAnalyser(); - analyser.fftSize = 2 ** 5; - source.connect(analyser); - const bufferLength = analyser.frequencyBinCount; - const dataArray = new Uint8Array(bufferLength); - - let animationId, previousTimeStamp; - const DATA_SIZE = 128.0; - const BUFFER_SIZE = 2 ** 8; - const dataBuffer = new Array(BUFFER_SIZE).fill(DATA_SIZE); - - const drawWaveform = (timeStamp) => { - if (!canvasCtx) return; - - analyser.getByteTimeDomainData(dataArray); - canvasCtx.clearRect(0, 0, canvas.width, canvas.height); - canvasCtx.fillStyle = "#cc3347"; - - if (previousTimeStamp === undefined) { - previousTimeStamp = timeStamp; - dataBuffer.push(Math.min(...dataArray)); - dataBuffer.splice(0, 1); - } - const elapsed = timeStamp - previousTimeStamp; - if (elapsed > 10) { - previousTimeStamp = timeStamp; - dataBuffer.push(Math.min(...dataArray)); - dataBuffer.splice(0, 1); - } - - // Drawing - const sliceWidth = canvas.width / dataBuffer.length; - let x = 0; - - for (let i = 0; i < dataBuffer.length; i++) { - const y = (canvas.height * dataBuffer[i]) / (2 * DATA_SIZE); - const sliceHeight = - ((1 - canvas.height) * dataBuffer[i]) / DATA_SIZE + canvas.height; - - canvasCtx.fillRect(x, y, (sliceWidth * 2) / 3, sliceHeight); - x += sliceWidth; - } - - animationId = requestAnimationFrame(drawWaveform); - }; - - drawWaveform(); - - return () => { - if (animationId) { - cancelAnimationFrame(animationId); - } - - if (source) { - source.disconnect(); - source.mediaStream.getTracks().forEach((track) => track.stop()); - } - - if (audioContext) { - audioContext.close(); - } - - canvas?.remove(); - }; - } - startRecording(stream) { - this.preventInteraction(); - this.cleanUp(); - - const onStop = this.render(stream); - const mediaRecorder = new MediaRecorder(stream, { - mimeType: this.options.mimeType || findSupportedMimeType(), - audioBitsPerSecond: this.options.audioBitsPerSecond, - }); - const recordedChunks = []; - - mediaRecorder.addEventListener("dataavailable", (event) => { - if (event.data.size > 0) { - recordedChunks.push(event.data); - } - }); - - mediaRecorder.addEventListener("stop", () => { - onStop(); - this.loadBlob(recordedChunks, mediaRecorder.mimeType); - this.emit("stopRecording"); - }); - - mediaRecorder.start(); - - this.emit("startRecording"); - - this.mediaRecorder = mediaRecorder; - } -} - -export default CustomRecordPlugin; diff --git a/www/app/transcripts/custom-plugins/regions.js b/www/app/transcripts/custom-plugins/regions.js deleted file mode 100644 index 528e99a3..00000000 --- a/www/app/transcripts/custom-plugins/regions.js +++ /dev/null @@ -1,12 +0,0 @@ -import RegionsPlugin from "wavesurfer.js/dist/plugins/regions"; - -class CustomRegionsPlugin extends RegionsPlugin { - static create(options) { - return new CustomRegionsPlugin(options); - } - avoidOverlapping(region) { - // Prevent overlapping regions - } -} - -export default CustomRegionsPlugin; From c970fa027afb5beb68cce8869c9a41c36269ca9b Mon Sep 17 00:00:00 2001 From: Jose B Date: Fri, 18 Aug 2023 16:19:46 -0500 Subject: [PATCH 10/15] more TS --- www/app/lib/custom-plugins/record.ts | 99 ++++++++++++++++--- .../transcripts/{recorder.js => recorder.tsx} | 72 ++++++++------ www/tsconfig.json | 3 +- 3 files changed, 132 insertions(+), 42 deletions(-) rename www/app/transcripts/{recorder.js => recorder.tsx} (77%) diff --git a/www/app/lib/custom-plugins/record.ts b/www/app/lib/custom-plugins/record.ts index aa85a812..2d42e80f 100644 --- a/www/app/lib/custom-plugins/record.ts +++ b/www/app/lib/custom-plugins/record.ts @@ -1,6 +1,21 @@ // Source code: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts +/** + * Record audio from the microphone, render a waveform and download the audio. + */ -import RecordPlugin from "wavesurfer.js/dist/plugins/record"; +import BasePlugin, { + type BasePluginEvents, +} from "wavesurfer.js/dist/base-plugin"; + +export type RecordPluginOptions = { + mimeType?: MediaRecorderOptions["mimeType"]; + audioBitsPerSecond?: MediaRecorderOptions["audioBitsPerSecond"]; +}; + +export type RecordPluginEvents = BasePluginEvents & { + startRecording: []; + stopRecording: []; +}; const MIME_TYPES = [ "audio/webm", @@ -12,11 +27,44 @@ const MIME_TYPES = [ const findSupportedMimeType = () => MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType)); -class CustomRecordPlugin extends RecordPlugin { - static create(options) { - return new CustomRecordPlugin(options || {}); +class RecordPlugin extends BasePlugin { + private mediaRecorder: MediaRecorder | null = null; + private recordedUrl = ""; + private savedCursorWidth = 1; + private savedInteractive = true; + + public static create(options?: RecordPluginOptions) { + return new RecordPlugin(options || {}); } - render(stream) { + + private preventInteraction() { + if (this.wavesurfer) { + this.savedCursorWidth = this.wavesurfer.options.cursorWidth || 1; + this.savedInteractive = this.wavesurfer.options.interact || true; + this.wavesurfer.options.cursorWidth = 0; + this.wavesurfer.options.interact = false; + } + } + + private restoreInteraction() { + if (this.wavesurfer) { + this.wavesurfer.options.cursorWidth = this.savedCursorWidth; + this.wavesurfer.options.interact = this.savedInteractive; + } + } + + onInit() { + this.preventInteraction(); + } + + private loadBlob(data: Blob[], type: string) { + const blob = new Blob(data, { type }); + this.recordedUrl = URL.createObjectURL(blob); + this.restoreInteraction(); + this.wavesurfer?.load(this.recordedUrl); + } + + render(stream: MediaStream): () => void { if (!this.wavesurfer) return () => undefined; const container = this.wavesurfer.getWrapper(); @@ -35,12 +83,12 @@ class CustomRecordPlugin extends RecordPlugin { const bufferLength = analyser.frequencyBinCount; const dataArray = new Uint8Array(bufferLength); - let animationId, previousTimeStamp; + let animationId: number, previousTimeStamp: number; const DATA_SIZE = 128.0; const BUFFER_SIZE = 2 ** 8; const dataBuffer = new Array(BUFFER_SIZE).fill(DATA_SIZE); - const drawWaveform = (timeStamp) => { + const drawWaveform = (timeStamp: number) => { if (!canvasCtx) return; analyser.getByteTimeDomainData(dataArray); @@ -75,7 +123,7 @@ class CustomRecordPlugin extends RecordPlugin { animationId = requestAnimationFrame(drawWaveform); }; - drawWaveform(); + drawWaveform(0); return () => { if (animationId) { @@ -94,7 +142,17 @@ class CustomRecordPlugin extends RecordPlugin { canvas?.remove(); }; } - startRecording(stream) { + + private cleanUp() { + this.stopRecording(); + this.wavesurfer?.empty(); + if (this.recordedUrl) { + URL.revokeObjectURL(this.recordedUrl); + this.recordedUrl = ""; + } + } + + public async startRecording(stream: MediaStream) { this.preventInteraction(); this.cleanUp(); @@ -103,7 +161,7 @@ class CustomRecordPlugin extends RecordPlugin { mimeType: this.options.mimeType || findSupportedMimeType(), audioBitsPerSecond: this.options.audioBitsPerSecond, }); - const recordedChunks = []; + const recordedChunks: Blob[] = []; mediaRecorder.addEventListener("dataavailable", (event) => { if (event.data.size > 0) { @@ -123,6 +181,25 @@ class CustomRecordPlugin extends RecordPlugin { this.mediaRecorder = mediaRecorder; } + + public isRecording(): boolean { + return this.mediaRecorder?.state === "recording"; + } + + public stopRecording() { + if (this.isRecording()) { + this.mediaRecorder?.stop(); + } + } + + public getRecordedUrl(): string { + return this.recordedUrl; + } + + public destroy() { + super.destroy(); + this.cleanUp(); + } } -export default CustomRecordPlugin; +export default RecordPlugin; diff --git a/www/app/transcripts/recorder.js b/www/app/transcripts/recorder.tsx similarity index 77% rename from www/app/transcripts/recorder.js rename to www/app/transcripts/recorder.tsx index 9ee68096..f7b7422c 100644 --- a/www/app/transcripts/recorder.js +++ b/www/app/transcripts/recorder.tsx @@ -1,19 +1,22 @@ import React, { useRef, useEffect, useState } from "react"; import WaveSurfer from "wavesurfer.js"; -import CustomRecordPlugin from "../lib/custom-plugins/record"; +import RecordPlugin from "../lib/custom-plugins/record"; import CustomRegionsPlugin from "../lib/custom-plugins/regions"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faDownload } from "@fortawesome/free-solid-svg-icons"; -import Dropdown from "react-dropdown"; +import Dropdown, { Option } from "react-dropdown"; import "react-dropdown/style.css"; import { formatTime } from "../lib/time"; -const AudioInputsDropdown = (props) => { - const [ddOptions, setDdOptions] = useState([]); +const AudioInputsDropdown: React.FC<{ + setDeviceId: React.Dispatch>; + disabled: boolean; +}> = (props) => { + const [ddOptions, setDdOptions] = useState>([]); useEffect(() => { const init = async () => { @@ -35,8 +38,8 @@ const AudioInputsDropdown = (props) => { init(); }, []); - const handleDropdownChange = (e) => { - props.setDeviceId(e.value); + const handleDropdownChange = (option: Option) => { + props.setDeviceId(option.value); }; return ( @@ -49,24 +52,27 @@ const AudioInputsDropdown = (props) => { ); }; -export default function Recorder(props) { +export default function Recorder(props: any) { const waveformRef = useRef(); - const [wavesurfer, setWavesurfer] = useState(null); - const [record, setRecord] = useState(null); - const [isRecording, setIsRecording] = useState(false); - const [isPlaying, setIsPlaying] = useState(false); - const [deviceId, setDeviceId] = useState(null); - const [currentTime, setCurrentTime] = useState(0); - const [timeInterval, setTimeInterval] = useState(null); - const [duration, setDuration] = useState(0); - const [waveRegions, setWaveRegions] = useState(null); + const [wavesurfer, setWavesurfer] = useState(null); + const [record, setRecord] = useState(null); + const [isRecording, setIsRecording] = useState(false); + const [isPlaying, setIsPlaying] = useState(false); + const [deviceId, setDeviceId] = useState(null); + const [currentTime, setCurrentTime] = useState(0); + const [timeInterval, setTimeInterval] = useState(null); + const [duration, setDuration] = useState(0); + const [waveRegions, setWaveRegions] = useState( + null, + ); const [activeTopic, setActiveTopic] = props.useActiveTopic; const topicsRef = useRef(props.topics); useEffect(() => { - document.getElementById("play-btn").disabled = true; + const playBtn = document.getElementById("play-btn"); + if (playBtn) playBtn.setAttribute("disabled", "true"); if (waveformRef.current) { const _wavesurfer = WaveSurfer.create({ @@ -92,7 +98,7 @@ export default function Recorder(props) { }); _wavesurfer.on("timeupdate", setCurrentTime); - setRecord(_wavesurfer.registerPlugin(CustomRecordPlugin.create())); + setRecord(_wavesurfer.registerPlugin(RecordPlugin.create())); setWaveRegions(_wavesurfer.registerPlugin(CustomRegionsPlugin.create())); setWavesurfer(_wavesurfer); @@ -115,7 +121,9 @@ export default function Recorder(props) { waveRegions.clearRegions(); for (let topic of topicsRef.current) { const content = document.createElement("div"); - content.style = ` + content.setAttribute( + "style", + ` position: absolute; border-left: solid 1px orange; padding: 0 2px 0 5px; @@ -129,15 +137,16 @@ export default function Recorder(props) { overflow: hidden; text-overflow: ellipsis; transition: width 100ms linear; - `; + `, + ); content.onmouseover = () => { content.style.backgroundColor = "orange"; - content.style.zIndex = 999; + content.style.zIndex = "999"; content.style.width = "300px"; }; content.onmouseout = () => { content.style.backgroundColor = "white"; - content.style.zIndex = 0; + content.style.zIndex = "0"; content.style.width = "100px"; }; content.textContent = topic.title; @@ -151,7 +160,7 @@ export default function Recorder(props) { region.on("click", (e) => { e.stopPropagation(); setActiveTopic(topic); - wavesurfer.setTime(region.start); + wavesurfer?.setTime(region.start); }); } }; @@ -160,8 +169,10 @@ export default function Recorder(props) { if (record) { return record.on("stopRecording", () => { const link = document.getElementById("download-recording"); - link.href = record.getRecordedUrl(); - link.download = "reflector-recording.webm"; + if (!link) return; + + link.setAttribute("href", record.getRecordedUrl()); + link.setAttribute("download", "reflector-recording.webm"); link.style.visibility = "visible"; renderMarkers(); }); @@ -170,13 +181,13 @@ export default function Recorder(props) { useEffect(() => { if (isRecording) { - const interval = setInterval(() => { + const interval = window.setInterval(() => { setCurrentTime((prev) => prev + 1); }, 1000); setTimeInterval(interval); return () => clearInterval(interval); } else { - clearInterval(timeInterval); + clearInterval(timeInterval as number); setCurrentTime((prev) => { setDuration(prev); return 0; @@ -186,7 +197,7 @@ export default function Recorder(props) { useEffect(() => { if (activeTopic) { - wavesurfer.setTime(activeTopic.timestamp); + wavesurfer?.setTime(activeTopic.timestamp); } }, [activeTopic]); @@ -197,11 +208,12 @@ export default function Recorder(props) { props.onStop(); record.stopRecording(); setIsRecording(false); - document.getElementById("play-btn").disabled = false; + const playBtn = document.getElementById("play-btn"); + if (playBtn) playBtn.removeAttribute("disabled"); } else { const stream = await navigator.mediaDevices.getUserMedia({ audio: { - deviceId, + deviceId: deviceId as string, noiseSuppression: false, echoCancellation: false, }, diff --git a/www/tsconfig.json b/www/tsconfig.json index 9c9b16c2..0e1b89ae 100644 --- a/www/tsconfig.json +++ b/www/tsconfig.json @@ -18,7 +18,8 @@ "name": "next" } ], - "strictNullChecks": true + "strictNullChecks": true, + "downlevelIteration": true }, "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"] From 213461cbb9df3860f7bb64da9ea48b414bd55543 Mon Sep 17 00:00:00 2001 From: Jose B Date: Fri, 18 Aug 2023 16:42:47 -0500 Subject: [PATCH 11/15] clear current time --- www/app/transcripts/recorder.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/www/app/transcripts/recorder.tsx b/www/app/transcripts/recorder.tsx index f7b7422c..19374b2f 100644 --- a/www/app/transcripts/recorder.tsx +++ b/www/app/transcripts/recorder.tsx @@ -106,6 +106,7 @@ export default function Recorder(props: any) { _wavesurfer.destroy(); setIsRecording(false); setIsPlaying(false); + setCurrentTime(0); }; } }, []); From 638729b4de79afc20814109ce7c8ac70d32097ad Mon Sep 17 00:00:00 2001 From: Jose B Date: Tue, 22 Aug 2023 13:35:55 -0500 Subject: [PATCH 12/15] fix class --- www/app/lib/custom-plugins/regions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/app/lib/custom-plugins/regions.ts b/www/app/lib/custom-plugins/regions.ts index 12fb7012..dff05f3b 100644 --- a/www/app/lib/custom-plugins/regions.ts +++ b/www/app/lib/custom-plugins/regions.ts @@ -6,7 +6,7 @@ import RegionsPlugin, { class CustomRegionsPlugin extends RegionsPlugin { public static create(options?: RegionsPluginOptions) { - return new RegionsPlugin(options); + return new CustomRegionsPlugin(options); } constructor(options?: RegionsPluginOptions) { From c755491ca41dae9cc593b162730b98195742a13e Mon Sep 17 00:00:00 2001 From: Jose B Date: Tue, 22 Aug 2023 13:41:06 -0500 Subject: [PATCH 13/15] fix build --- www/app/transcripts/recorder.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/app/transcripts/recorder.tsx b/www/app/transcripts/recorder.tsx index 19374b2f..6d0731ac 100644 --- a/www/app/transcripts/recorder.tsx +++ b/www/app/transcripts/recorder.tsx @@ -53,7 +53,7 @@ const AudioInputsDropdown: React.FC<{ }; export default function Recorder(props: any) { - const waveformRef = useRef(); + const waveformRef = useRef(null); const [wavesurfer, setWavesurfer] = useState(null); const [record, setRecord] = useState(null); const [isRecording, setIsRecording] = useState(false); From 92cd95057202e7cc1ef29a2fb6c53bf58ca2ca4d Mon Sep 17 00:00:00 2001 From: Koper Date: Wed, 23 Aug 2023 15:35:59 +0700 Subject: [PATCH 14/15] Force vercel re-deploy --- server/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/README.md b/server/README.md index b0ca34b7..b01c9253 100644 --- a/server/README.md +++ b/server/README.md @@ -225,7 +225,7 @@ From the reflector root folder, run `python3 whisjax_realtime.py` -The transcription text should be written to `real_time_transcription_.txt`. +The transcription text should be written to `real_time_transcription_.txt` NEXT STEPS: From 56b554f85fa30472c6ea82675b0576425c99aadf Mon Sep 17 00:00:00 2001 From: Koper Date: Wed, 23 Aug 2023 15:38:14 +0700 Subject: [PATCH 15/15] Update README.md --- www/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/README.md b/www/README.md index c92df561..03764cd4 100644 --- a/www/README.md +++ b/www/README.md @@ -78,7 +78,7 @@ This data is then returned from the `useWebRTC` hook and can be used in your com To generate the TypeScript files from the openapi.json file, make sure the python server is running, then run: -``` +```bash yarn openapi ```