Merge pull request #9 from Monadical-SAS/jose/vertical-waveform

Improve recording waveform speed + query permissions

app/components/CustomRecordPlugin.js

@@ -17,6 +17,83 @@ class CustomRecordPlugin extends RecordPlugin {
   static create(options) {
     return new CustomRecordPlugin(options || {});
   }
+  render(stream) {
+    if (!this.wavesurfer) return () => undefined
+
+    const container = this.wavesurfer.getWrapper()
+    const canvas = document.createElement('canvas')
+    canvas.width = container.clientWidth
+    canvas.height = container.clientHeight
+    canvas.style.zIndex = '10'
+    container.appendChild(canvas)
+
+    const canvasCtx = canvas.getContext('2d')
+    const audioContext = new AudioContext()
+    const source = audioContext.createMediaStreamSource(stream)
+    const analyser = audioContext.createAnalyser()
+    analyser.fftSize = 2 ** 5
+    source.connect(analyser)
+    const bufferLength = analyser.frequencyBinCount
+    const dataArray = new Uint8Array(bufferLength)
+
+    let animationId, previousTimeStamp;
+    const BUFFER_SIZE = 2 ** 8
+    const dataBuffer = new Array(BUFFER_SIZE).fill(canvas.height)
+
+    const drawWaveform = (timeStamp) => {
+      if (!canvasCtx) return
+
+      analyser.getByteTimeDomainData(dataArray)
+      canvasCtx.clearRect(0, 0, canvas.width, canvas.height)
+      canvasCtx.fillStyle = 'black'
+
+      if (previousTimeStamp === undefined) {
+        previousTimeStamp = timeStamp
+        dataBuffer.push(Math.min(...dataArray))
+        dataBuffer.splice(0, 1)
+      }
+      const elapsed = timeStamp - previousTimeStamp;
+      if (elapsed > 10) {
+        previousTimeStamp = timeStamp
+        dataBuffer.push(Math.min(...dataArray))
+        dataBuffer.splice(0, 1)
+      }
+
+      // Drawing
+      const sliceWidth = canvas.width / dataBuffer.length
+      let x = 0
+
+      for (let i = 0; i < dataBuffer.length; i++) {
+        const valueNormalized = dataBuffer[i] / canvas.height
+        const y = valueNormalized * canvas.height / 2
+        const sliceHeight = canvas.height + 1 - y * 2
+
+        canvasCtx.fillRect(x, y, sliceWidth * 2 / 3, sliceHeight)
+        x += sliceWidth
+      }
+
+      animationId = requestAnimationFrame(drawWaveform)
+    }
+
+    drawWaveform()
+
+    return () => {
+      if (animationId) {
+        cancelAnimationFrame(animationId)
+      }
+
+      if (source) {
+        source.disconnect()
+        source.mediaStream.getTracks().forEach((track) => track.stop())
+      }
+
+      if (audioContext) {
+        audioContext.close()
+      }
+
+      canvas?.remove()
+    }
+  }
   startRecording(stream) {
     this.preventInteraction();
     this.cleanUp();

app/components/record.js

@@ -7,6 +7,47 @@ import "react-dropdown/style.css";
 
 import CustomRecordPlugin from "./CustomRecordPlugin";
 
+const queryAndPromptAudio = async () => {
+  const permissionStatus = await navigator.permissions.query({name: 'microphone'})
+  if (permissionStatus.state == 'prompt') {
+    await navigator.mediaDevices.getUserMedia({ audio: true })
+  }
+}
+
+const AudioInputsDropdown = (props) => {
+  const [ddOptions, setDdOptions] = useState([]);
+
+  useEffect(() => {
+    const init = async () => {
+      await queryAndPromptAudio()
+
+      const devices = await navigator.mediaDevices.enumerateDevices()
+      const audioDevices = devices
+        .filter((d) => d.kind === "audioinput" && d.deviceId != "")
+        .map((d) => ({ value: d.deviceId, label: d.label }))
+
+      if (audioDevices.length < 1) return console.log("no audio input devices")
+
+      setDdOptions(audioDevices)
+      props.setDeviceId(audioDevices[0].value)
+    }
+    init()
+  }, [])
+
+  const handleDropdownChange = (e) => {
+    props.setDeviceId(e.value);
+  };
+
+  return (
+    <Dropdown
+      options={ddOptions}
+      onChange={handleDropdownChange}
+      value={ddOptions[0]}
+      disabled={props.disabled}
+    />
+  )
+}
+
 export default function Recorder(props) {
   const waveformRef = useRef();
   const [wavesurfer, setWavesurfer] = useState(null);
@@ -14,22 +55,10 @@ export default function Recorder(props) {
   const [isRecording, setIsRecording] = useState(false);
   const [isPlaying, setIsPlaying] = useState(false);
   const [deviceId, setDeviceId] = useState(null);
-  const [ddOptions, setDdOptions] = useState([]);
 
   useEffect(() => {
     document.getElementById("play-btn").disabled = true;
 
-    navigator.mediaDevices.enumerateDevices().then((devices) => {
-      const audioDevices = devices
-        .filter((d) => d.kind === "audioinput")
-        .map((d) => ({ value: d.deviceId, label: d.label }));
-
-      if (audioDevices.length < 1) return console.log("no audio input devices");
-
-      setDdOptions(audioDevices);
-      setDeviceId(audioDevices[0].value);
-    });
-
     if (waveformRef.current) {
       const _wavesurfer = WaveSurfer.create({
         container: waveformRef.current,
@@ -85,22 +114,15 @@ export default function Recorder(props) {
     wavesurfer?.playPause();
   };
 
-  const handleDropdownChange = (e) => {
-    setDeviceId(e.value);
-  };
-
   return (
     <div className="flex flex-col items-center justify-center max-w-[75vw] w-full">
       <div className="flex my-2 mx-auto">
-        <Dropdown
-          options={ddOptions}
-          onChange={handleDropdownChange}
-          value={ddOptions[0]}
-        />
+        <AudioInputsDropdown setDeviceId={setDeviceId} disabled={isRecording} />
         &nbsp;
         <button
           onClick={handleRecClick}
           data-color={isRecording ? "red" : "blue"}
+          disabled={!deviceId}
         >
           {isRecording ? "Stop" : "Record"}
         </button>

app/page.js

@@ -6,23 +6,11 @@ import useWebRTC from "./components/webrtc.js";
 import "../public/button.css";
 
 const App = () => {
-  const [isRecording, setIsRecording] = useState(false);
   const [stream, setStream] = useState(null);
 
-  const handleRecord = (recording) => {
-    setIsRecording(recording);
-
-    if (recording) {
-      navigator.mediaDevices
-        .getUserMedia({ audio: true })
-        .then(setStream)
-        .catch((err) => console.error(err));
-    } else if (!recording && serverData.peer) {
-      serverData.peer.send(JSON.stringify({ cmd: "STOP" }));
-    }
-  };
-
-  const serverData = useWebRTC(stream, setIsRecording);
+  // This is where you'd send the stream and receive the data from the server.
+  // transcription, summary, etc
+  const serverData = useWebRTC(stream, () => {});
 
   return (
     <div className="flex flex-col items-center h-[100svh]">
@@ -33,8 +21,6 @@ const App = () => {
 
       <Recorder setStream={setStream} serverData={serverData} />
       <Dashboard
-        isRecording={isRecording}
-        onRecord={(recording) => handleRecord(recording)}
         transcriptionText={serverData.text ?? "..."}
         finalSummary={serverData.finalSummary}
         topics={serverData.topics ?? []}