From 2420e18ba333392922b79dda5bea1d7874cff6f9 Mon Sep 17 00:00:00 2001 From: Koper Date: Wed, 26 Jul 2023 16:38:54 +0700 Subject: [PATCH 1/4] Improve recording waveform speed + query permissions (by Jose) --- www/app/components/CustomRecordPlugin.js | 77 ++++++++++++++++++++++++ www/app/components/record.js | 64 +++++++++++++------- www/app/page.js | 20 +----- 3 files changed, 123 insertions(+), 38 deletions(-) diff --git a/www/app/components/CustomRecordPlugin.js b/www/app/components/CustomRecordPlugin.js index bbe21195..b6f4fd88 100644 --- a/www/app/components/CustomRecordPlugin.js +++ b/www/app/components/CustomRecordPlugin.js @@ -17,6 +17,83 @@ class CustomRecordPlugin extends RecordPlugin { static create(options) { return new CustomRecordPlugin(options || {}); } + render(stream) { + if (!this.wavesurfer) return () => undefined + + const container = this.wavesurfer.getWrapper() + const canvas = document.createElement('canvas') + canvas.width = container.clientWidth + canvas.height = container.clientHeight + canvas.style.zIndex = '10' + container.appendChild(canvas) + + const canvasCtx = canvas.getContext('2d') + const audioContext = new AudioContext() + const source = audioContext.createMediaStreamSource(stream) + const analyser = audioContext.createAnalyser() + analyser.fftSize = 2 ** 5 + source.connect(analyser) + const bufferLength = analyser.frequencyBinCount + const dataArray = new Uint8Array(bufferLength) + + let animationId, previousTimeStamp; + const BUFFER_SIZE = 2 ** 8 + const dataBuffer = new Array(BUFFER_SIZE).fill(canvas.height) + + const drawWaveform = (timeStamp) => { + if (!canvasCtx) return + + analyser.getByteTimeDomainData(dataArray) + canvasCtx.clearRect(0, 0, canvas.width, canvas.height) + canvasCtx.fillStyle = 'black' + + if (previousTimeStamp === undefined) { + previousTimeStamp = timeStamp + dataBuffer.push(Math.min(...dataArray)) + dataBuffer.splice(0, 1) + } + const elapsed = timeStamp - previousTimeStamp; + if (elapsed > 10) { + previousTimeStamp = timeStamp + dataBuffer.push(Math.min(...dataArray)) + dataBuffer.splice(0, 1) + } + + // Drawing + const sliceWidth = canvas.width / dataBuffer.length + let x = 0 + + for (let i = 0; i < dataBuffer.length; i++) { + const valueNormalized = dataBuffer[i] / canvas.height + const y = valueNormalized * canvas.height / 2 + const sliceHeight = canvas.height + 1 - y * 2 + + canvasCtx.fillRect(x, y, sliceWidth * 2 / 3, sliceHeight) + x += sliceWidth + } + + animationId = requestAnimationFrame(drawWaveform) + } + + drawWaveform() + + return () => { + if (animationId) { + cancelAnimationFrame(animationId) + } + + if (source) { + source.disconnect() + source.mediaStream.getTracks().forEach((track) => track.stop()) + } + + if (audioContext) { + audioContext.close() + } + + canvas?.remove() + } + } startRecording(stream) { this.preventInteraction(); this.cleanUp(); diff --git a/www/app/components/record.js b/www/app/components/record.js index 70336e5d..3b697ad4 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -7,6 +7,47 @@ import "react-dropdown/style.css"; import CustomRecordPlugin from "./CustomRecordPlugin"; +const queryAndPromptAudio = async () => { + const permissionStatus = await navigator.permissions.query({ name: 'microphone' }) + if (permissionStatus.state == 'prompt') { + await navigator.mediaDevices.getUserMedia({ audio: true }) + } +} + +const AudioInputsDropdown = (props) => { + const [ddOptions, setDdOptions] = useState([]); + + useEffect(() => { + const init = async () => { + await queryAndPromptAudio() + + const devices = await navigator.mediaDevices.enumerateDevices() + const audioDevices = devices + .filter((d) => d.kind === "audioinput" && d.deviceId != "") + .map((d) => ({ value: d.deviceId, label: d.label })) + + if (audioDevices.length < 1) return console.log("no audio input devices") + + setDdOptions(audioDevices) + props.setDeviceId(audioDevices[0].value) + } + init() + }, []) + + const handleDropdownChange = (e) => { + props.setDeviceId(e.value); + }; + + return ( + + ) +} + export default function Recorder(props) { const waveformRef = useRef(); const [wavesurfer, setWavesurfer] = useState(null); @@ -14,22 +55,10 @@ export default function Recorder(props) { const [isRecording, setIsRecording] = useState(false); const [isPlaying, setIsPlaying] = useState(false); const [deviceId, setDeviceId] = useState(null); - const [ddOptions, setDdOptions] = useState([]); useEffect(() => { document.getElementById("play-btn").disabled = true; - navigator.mediaDevices.enumerateDevices().then((devices) => { - const audioDevices = devices - .filter((d) => d.kind === "audioinput") - .map((d) => ({ value: d.deviceId, label: d.label })); - - if (audioDevices.length < 1) return console.log("no audio input devices"); - - setDdOptions(audioDevices); - setDeviceId(audioDevices[0].value); - }); - if (waveformRef.current) { const _wavesurfer = WaveSurfer.create({ container: waveformRef.current, @@ -85,22 +114,15 @@ export default function Recorder(props) { wavesurfer?.playPause(); }; - const handleDropdownChange = (e) => { - setDeviceId(e.value); - }; - return (
- +   diff --git a/www/app/page.js b/www/app/page.js index 2e4bfc93..635caa5e 100644 --- a/www/app/page.js +++ b/www/app/page.js @@ -6,23 +6,11 @@ import useWebRTC from "./components/webrtc.js"; import "../public/button.css"; const App = () => { - const [isRecording, setIsRecording] = useState(false); const [stream, setStream] = useState(null); - const handleRecord = (recording) => { - setIsRecording(recording); - - if (recording) { - navigator.mediaDevices - .getUserMedia({ audio: true }) - .then(setStream) - .catch((err) => console.error(err)); - } else if (!recording && serverData.peer) { - serverData.peer.send(JSON.stringify({ cmd: "STOP" })); - } - }; - - const serverData = useWebRTC(stream, setIsRecording); + // This is where you'd send the stream and receive the data from the server. + // transcription, summary, etc + const serverData = useWebRTC(stream, setIsRecording); return (
@@ -33,8 +21,6 @@ const App = () => { handleRecord(recording)} transcriptionText={serverData.text ?? "(No transcription yet)"} finalSummary={serverData.finalSummary} topics={serverData.topics ?? []} From 1836ad8e02221d1495fea8d867d68524169e3ea8 Mon Sep 17 00:00:00 2001 From: Koper Date: Wed, 26 Jul 2023 16:43:12 +0700 Subject: [PATCH 2/4] Update page.js --- www/app/page.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/app/page.js b/www/app/page.js index 635caa5e..006be1b2 100644 --- a/www/app/page.js +++ b/www/app/page.js @@ -10,7 +10,7 @@ const App = () => { // This is where you'd send the stream and receive the data from the server. // transcription, summary, etc - const serverData = useWebRTC(stream, setIsRecording); + const serverData = useWebRTC(stream, () => { }); return (
From 05c5f6aefe1382dcf22aa0d7d888d22f86b112f9 Mon Sep 17 00:00:00 2001 From: Jose B Date: Wed, 26 Jul 2023 23:38:57 -0500 Subject: [PATCH 3/4] update colors + tweak code --- www/app/components/CustomRecordPlugin.js | 2 +- www/app/components/record.js | 16 +++++----------- www/app/components/webrtc.js | 2 +- www/app/page.js | 4 +++- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/www/app/components/CustomRecordPlugin.js b/www/app/components/CustomRecordPlugin.js index b6f4fd88..8e8cdc44 100644 --- a/www/app/components/CustomRecordPlugin.js +++ b/www/app/components/CustomRecordPlugin.js @@ -45,7 +45,7 @@ class CustomRecordPlugin extends RecordPlugin { analyser.getByteTimeDomainData(dataArray) canvasCtx.clearRect(0, 0, canvas.width, canvas.height) - canvasCtx.fillStyle = 'black' + canvasCtx.fillStyle = '#cc3347' if (previousTimeStamp === undefined) { previousTimeStamp = timeStamp diff --git a/www/app/components/record.js b/www/app/components/record.js index 3b697ad4..eb8f5912 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -7,19 +7,14 @@ import "react-dropdown/style.css"; import CustomRecordPlugin from "./CustomRecordPlugin"; -const queryAndPromptAudio = async () => { - const permissionStatus = await navigator.permissions.query({ name: 'microphone' }) - if (permissionStatus.state == 'prompt') { - await navigator.mediaDevices.getUserMedia({ audio: true }) - } -} const AudioInputsDropdown = (props) => { const [ddOptions, setDdOptions] = useState([]); useEffect(() => { const init = async () => { - await queryAndPromptAudio() + // Request permission to use audio inputs + await navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => stream.getTracks().forEach((t) => t.stop())) const devices = await navigator.mediaDevices.enumerateDevices() const audioDevices = devices @@ -62,8 +57,8 @@ export default function Recorder(props) { if (waveformRef.current) { const _wavesurfer = WaveSurfer.create({ container: waveformRef.current, - waveColor: "#cc3347", - progressColor: "#0178FFπ", + waveColor: "#777", + progressColor: "#222", cursorColor: "OrangeRed", hideScrollbar: true, autoCenter: true, @@ -95,8 +90,7 @@ export default function Recorder(props) { if (!record) return console.log("no record"); if (record?.isRecording()) { - - props.serverData.peer.send(JSON.stringify({ cmd: "STOP" })); + props.onStop(); record.stopRecording(); setIsRecording(false); document.getElementById("play-btn").disabled = false; diff --git a/www/app/components/webrtc.js b/www/app/components/webrtc.js index 41759e69..0b8e21b6 100644 --- a/www/app/components/webrtc.js +++ b/www/app/components/webrtc.js @@ -30,7 +30,7 @@ const useWebRTC = (stream, setIsRecording) => { .then((response) => response.json()) .then((answer) => peer.signal(answer)) .catch((e) => { - alert(e); + console.log("Error signaling:", e); }); } }); diff --git a/www/app/page.js b/www/app/page.js index 006be1b2..9aba71a7 100644 --- a/www/app/page.js +++ b/www/app/page.js @@ -12,6 +12,8 @@ const App = () => { // transcription, summary, etc const serverData = useWebRTC(stream, () => { }); + const sendStopCmd = () => serverData?.peer?.send(JSON.stringify({ cmd: "STOP" })) + return (
@@ -19,7 +21,7 @@ const App = () => {

Capture The Signal, Not The Noise

- + Date: Thu, 27 Jul 2023 19:42:09 -0500 Subject: [PATCH 4/4] no need to stop from server --- www/app/components/record.js | 2 +- www/app/components/webrtc.js | 5 ++--- www/app/page.js | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/www/app/components/record.js b/www/app/components/record.js index eb8f5912..cb3b03f8 100644 --- a/www/app/components/record.js +++ b/www/app/components/record.js @@ -89,7 +89,7 @@ export default function Recorder(props) { const handleRecClick = async () => { if (!record) return console.log("no record"); - if (record?.isRecording()) { + if (record.isRecording()) { props.onStop(); record.stopRecording(); setIsRecording(false); diff --git a/www/app/components/webrtc.js b/www/app/components/webrtc.js index 0b8e21b6..1b9843c0 100644 --- a/www/app/components/webrtc.js +++ b/www/app/components/webrtc.js @@ -3,7 +3,7 @@ import Peer from "simple-peer"; const WebRTC_SERVER_URL = "http://127.0.0.1:1250/offer"; -const useWebRTC = (stream, setIsRecording) => { +const useWebRTC = (stream) => { const [data, setData] = useState({ peer: null, }); @@ -66,7 +66,6 @@ const useWebRTC = (stream, setIsRecording) => { }, text: '' })); - setIsRecording(false); break; default: console.error(`Unknown command ${serverData.cmd}`); @@ -76,7 +75,7 @@ const useWebRTC = (stream, setIsRecording) => { return () => { peer.destroy(); }; - }, [stream, setIsRecording]); + }, [stream]); return data; }; diff --git a/www/app/page.js b/www/app/page.js index 9aba71a7..6dae28ea 100644 --- a/www/app/page.js +++ b/www/app/page.js @@ -10,7 +10,7 @@ const App = () => { // This is where you'd send the stream and receive the data from the server. // transcription, summary, etc - const serverData = useWebRTC(stream, () => { }); + const serverData = useWebRTC(stream); const sendStopCmd = () => serverData?.peer?.send(JSON.stringify({ cmd: "STOP" }))