Merge pull request #4 from Monadical-SAS/jose/vertical-waveform

Wavesurfer implementation
2025-12-20 20:29:06 +00:00 · 2023-07-21 18:09:25 +07:00
parent 8168e13dd7 f4235147aa
commit d3165b7b0a
10 changed files with 233 additions and 139 deletions
--- a/app/components/CustomRecordPlugin.js
+++ b/app/components/CustomRecordPlugin.js
@@ -0,0 +1,51 @@
+// Override the startRecording method so we can pass the desired stream
+// Checkout: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts
+
+import RecordPlugin from "wavesurfer.js/dist/plugins/record";
+
+const MIME_TYPES = [
+  "audio/webm",
+  "audio/wav",
+  "audio/mpeg",
+  "audio/mp4",
+  "audio/mp3",
+];
+const findSupportedMimeType = () =>
+  MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType));
+
+class CustomRecordPlugin extends RecordPlugin {
+  static create(options) {
+    return new CustomRecordPlugin(options || {});
+  }
+  startRecording(stream) {
+    this.preventInteraction();
+    this.cleanUp();
+
+    const onStop = this.render(stream);
+    const mediaRecorder = new MediaRecorder(stream, {
+      mimeType: this.options.mimeType || findSupportedMimeType(),
+      audioBitsPerSecond: this.options.audioBitsPerSecond,
+    });
+    const recordedChunks = [];
+
+    mediaRecorder.addEventListener("dataavailable", (event) => {
+      if (event.data.size > 0) {
+        recordedChunks.push(event.data);
+      }
+    });
+
+    mediaRecorder.addEventListener("stop", () => {
+      onStop();
+      this.loadBlob(recordedChunks, mediaRecorder.mimeType);
+      this.emit("stopRecording");
+    });
+
+    mediaRecorder.start();
+
+    this.emit("startRecording");
+
+    this.mediaRecorder = mediaRecorder;
+  }
+}
+
+export default CustomRecordPlugin;
--- a/app/components/audioVisualizer.js
+++ b/app/components/audioVisualizer.js
@@ -1,66 +0,0 @@
-import React, { useRef, useEffect } from "react";
-
-function AudioVisualizer(props) {
-  const canvasRef = useRef(null);
-
-  useEffect(() => {
-    let animationFrameId;
-
-    const canvas = canvasRef.current;
-    const context = canvas.getContext("2d");
-    const analyser = new AnalyserNode(new AudioContext());
-
-    navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
-      const audioContext = new (window.AudioContext ||
-        window.webkitAudioContext)();
-      const source = audioContext.createMediaStreamSource(stream);
-      const analyser = audioContext.createAnalyser();
-      analyser.fftSize = 2048;
-      source.connect(analyser);
-
-      const bufferLength = analyser.frequencyBinCount;
-      const dataArray = new Uint8Array(bufferLength);
-      const barWidth = (canvas.width / bufferLength) * 2.5;
-      let barHeight;
-      let x = 0;
-
-      function renderFrame() {
-        x = 0;
-        analyser.getByteFrequencyData(dataArray);
-        context.fillStyle = "#000";
-        context.fillRect(0, 0, canvas.width, canvas.height);
-
-        for (let i = 0; i < bufferLength; i++) {
-          barHeight = dataArray[i];
-
-          const red = 255;
-          const green = 250 * (i / bufferLength);
-          const blue = barHeight + 25 * (i / bufferLength);
-
-          context.fillStyle = `rgb(${red},${green},${blue})`;
-          context.fillRect(
-            x,
-            canvas.height - barHeight / 2,
-            barWidth,
-            barHeight / 2,
-          );
-
-          x += barWidth + 1;
-        }
-        animationFrameId = requestAnimationFrame(renderFrame);
-      }
-      renderFrame();
-    });
-
-    return () => cancelAnimationFrame(animationFrameId);
-  }, []);
-
-  return (
-    <>
-      <p>Is recording: {props.isRecording ? "true" : "false"}</p>
-      <canvas className="w-full h-16" ref={canvasRef} />
-    </>
-  );
-}
-
-export default AudioVisualizer;
--- a/app/components/dashboard.js
+++ b/app/components/dashboard.js
@@ -1,6 +1,5 @@
 import { Mulberry32 } from "../utils.js";
 import React, { useState, useEffect } from "react";
-import AudioVisualizer from "./audioVisualizer.js";

 export function Dashboard({
  isRecording,
@@ -17,8 +16,7 @@ export function Dashboard({
    <>
      <div className="w-3/4 py-4">
        <div className="text-center py-6">
-          <h1 className="text-4xl font-bold text-blue-500">Reflector</h1>
-          <p className="text-gray-500">Capture The Signal, Not The Noise</p>
+          <h1 className="text-2xl font-bold text-blue-500">Meeting Topics</h1>
        </div>
        <div className="flex justify-between border-b-2">
          <div className="w-1/4">Timestamp</div>
@@ -43,8 +41,7 @@ export function Dashboard({
                  {">"}
                </span>
              </div>
-              <div className="w-1/4 flex flex-row space-x-0.5">
-              </div>
+              <div className="w-1/4 flex flex-row space-x-0.5"></div>
            </div>
            {openIndex === index && (
              <div className="mt-2 p-2 bg-white">{item.transcript}</div>
@@ -56,34 +53,12 @@ export function Dashboard({
          <div className="flex justify-between">
            <div className="w-1/4">Live</div>
            <div className="w-1/4">Transcript</div>
-            <div className="w-1/4 flex flex-row space-x-0.5">
-            </div>
+            <div className="w-1/4 flex flex-row space-x-0.5"></div>
          </div>
          <div className="mt-2 p-2 bg-white temp-transcription">
            {transcriptionText}
          </div>
        </div>
-        <AudioVisualizer isRecording={isRecording} />
-
-        <button
-          className="mx-auto mt-6 mb-9"
-          onClick={() => onRecord(!isRecording)}
-          data-color={isRecording ? "red" : "blue"}
-        >
-          {isRecording ? "STOP" : "RESUME"}
-        </button>
-
-        {finalSummary && (
-          <div>
-            <h2>Final Summary</h2>
-            <p>Duration: {finalSummary.duration}</p>
-            <p>{finalSummary.summary}</p>
-          </div>
-        )}
-
-        <footer className="w-full bg-gray-800 text-center py-4 mt-4 text-white">
-          Reflector © 2023 Monadical
-        </footer>
      </div>
    </>
  );
--- a/app/components/record.js
+++ b/app/components/record.js
@@ -1,22 +1,118 @@
-export default function Record(props) {
-  return (
-    <div className="flex flex-col justify-center h-screen">
-      <div className="text-center py-6 mt-10">
-        <h1 className="text-5xl font-bold text-blue-500">Reflector</h1>
-        <p className="text-gray-500">Capture The Signal, Not The Noise</p>
-      </div>
+import React, { useRef, useEffect, useState } from "react";

-      <div className="flex flex-col items-center justify-center flex-grow -mt-10">
-        {!props.isRecording ? (
-          <button onClick={() => props.onRecord(true)} data-color="blue">
-            Record
-          </button>
-        ) : (
-          <button onClick={() => props.onRecord(false)} data-color="red">
-            Stop
-          </button>
-        )}
+import WaveSurfer from "wavesurfer.js";
+
+import Dropdown from "react-dropdown";
+import "react-dropdown/style.css";
+
+import CustomRecordPlugin from "./CustomRecordPlugin";
+
+export default function Recorder(props) {
+  const waveformRef = useRef();
+  const [wavesurfer, setWavesurfer] = useState(null);
+  const [record, setRecord] = useState(null);
+  const [isRecording, setIsRecording] = useState(false);
+  const [isPlaying, setIsPlaying] = useState(false);
+  const [deviceId, setDeviceId] = useState(null);
+  const [ddOptions, setDdOptions] = useState([]);
+
+  useEffect(() => {
+    document.getElementById("play-btn").disabled = true;
+
+    navigator.mediaDevices.enumerateDevices().then((devices) => {
+      const audioDevices = devices
+        .filter((d) => d.kind === "audioinput")
+        .map((d) => ({ value: d.deviceId, label: d.label }));
+
+      if (audioDevices.length < 1) return console.log("no audio input devices");
+
+      setDdOptions(audioDevices);
+      setDeviceId(audioDevices[0].value);
+    });
+
+    if (waveformRef.current) {
+      const _wavesurfer = WaveSurfer.create({
+        container: waveformRef.current,
+        waveColor: "#cc3347",
+        progressColor: "#0178FFπ",
+        cursorColor: "OrangeRed",
+        hideScrollbar: true,
+        autoCenter: true,
+        barWidth: 2,
+      });
+      const wsWrapper = _wavesurfer.getWrapper();
+      wsWrapper.style.cursor = "pointer";
+      wsWrapper.style.backgroundColor = "lightgray";
+      wsWrapper.style.borderRadius = "15px";
+
+      _wavesurfer.on("play", () => {
+        setIsPlaying(true);
+      });
+      _wavesurfer.on("pause", () => {
+        setIsPlaying(false);
+      });
+
+      setRecord(_wavesurfer.registerPlugin(CustomRecordPlugin.create()));
+      setWavesurfer(_wavesurfer);
+      return () => {
+        _wavesurfer.destroy();
+        setIsRecording(false);
+        setIsPlaying(false);
+      };
+    }
+  }, []);
+
+  const handleRecClick = async () => {
+    if (!record) return console.log("no record");
+
+    if (record?.isRecording()) {
+      record.stopRecording();
+      setIsRecording(false);
+      document.getElementById("play-btn").disabled = false;
+    } else {
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: { deviceId },
+      });
+      await record.startRecording(stream);
+      props.setStream(stream);
+      setIsRecording(true);
+    }
+  };
+
+  const handlePlayClick = () => {
+    wavesurfer?.playPause();
+  };
+
+  const handleDropdownChange = (e) => {
+    setDeviceId(e.value);
+  };
+
+  return (
+    <div className="flex flex-col items-center justify-center max-w-[75vw] w-full">
+      <div className="flex my-2 mx-auto">
+        <Dropdown
+          options={ddOptions}
+          onChange={handleDropdownChange}
+          value={ddOptions[0]}
+        />
+        &nbsp;
+        <button
+          onClick={handleRecClick}
+          data-color={isRecording ? "red" : "blue"}
+        >
+          {isRecording ? "Stop" : "Record"}
+        </button>
+        &nbsp;
+        <button
+          id="play-btn"
+          onClick={handlePlayClick}
+          data-color={isPlaying ? "orange" : "green"}
+        >
+          {isPlaying ? "Pause" : "Play"}
+        </button>
      </div>
+      <div ref={waveformRef} className="w-full"></div>
+      {/* TODO: Download audio <a> tag */}
    </div>
  );
 }
--- a/app/components/webrtc.js
+++ b/app/components/webrtc.js
@@ -37,7 +37,7 @@ const useWebRTC = (stream, setIsRecording) => {

    peer.on("connect", () => {
      console.log("WebRTC connected");
-      setData(prevData => ({ ...prevData, peer: peer }));
+      setData((prevData) => ({ ...prevData, peer: peer }));
    });

    peer.on("data", (data) => {
--- a/app/page.js
+++ b/app/page.js
@@ -1,18 +1,16 @@
 "use client";
 import React, { useState } from "react";
-import Record from "./components/record.js";
+import Recorder from "./components/record.js";
 import { Dashboard } from "./components/dashboard.js";
 import useWebRTC from "./components/webrtc.js";
 import "../public/button.css";

 const App = () => {
  const [isRecording, setIsRecording] = useState(false);
-  const [splashScreen, setSplashScreen] = useState(true);
  const [stream, setStream] = useState(null);

  const handleRecord = (recording) => {
    setIsRecording(recording);
-    setSplashScreen(false);

    if (recording) {
      navigator.mediaDevices
@@ -20,31 +18,28 @@ const App = () => {
        .then(setStream)
        .catch((err) => console.error(err));
    } else if (!recording && serverData.peer) {
-      serverData.peer.send(JSON.stringify({ cmd: 'STOP' }));
+      serverData.peer.send(JSON.stringify({ cmd: "STOP" }));
    }
  };

-
  const serverData = useWebRTC(stream, setIsRecording);

  return (
-    <div className="flex flex-col items-center justify-center min-h-screen bg-gray-100">
-      {splashScreen && (
-        <Record
-          isRecording={isRecording}
-          onRecord={(recording) => handleRecord(recording)}
-        />
-      )}
-      {!splashScreen && (
-        <Dashboard
-          isRecording={isRecording}
-          onRecord={(recording) => handleRecord(recording)}
-          transcriptionText={serverData.text ?? "(No transcription text)"}
-          finalSummary={serverData.finalSummary}
-          topics={serverData.topics ?? []}
-          stream={stream}
-        />
-      )}
+    <div className="flex flex-col items-center h-[100svh]">
+      <div className="text-center py-6 mt-10">
+        <h1 className="text-5xl font-bold text-blue-500">Reflector</h1>
+        <p className="text-gray-500">Capture The Signal, Not The Noise</p>
+      </div>
+
+      <Recorder setStream={setStream} />
+      <Dashboard
+        isRecording={isRecording}
+        onRecord={(recording) => handleRecord(recording)}
+        transcriptionText={serverData.text ?? "(No transcription text)"}
+        finalSummary={serverData.finalSummary}
+        topics={serverData.topics ?? []}
+        stream={stream}
+      />
    </div>
  );
 };
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,10 +14,12 @@
        "postcss": "8.4.25",
        "react": "^18.2.0",
        "react-dom": "^18.2.0",
+        "react-dropdown": "^1.11.0",
        "sass": "^1.63.6",
        "simple-peer": "^9.11.1",
        "supports-color": "^9.4.0",
-        "tailwindcss": "^3.3.2"
+        "tailwindcss": "^3.3.2",
+        "wavesurfer.js": "^7.0.3"
      },
      "devDependencies": {
        "prettier": "^3.0.0"
@@ -497,6 +499,11 @@
        "node": ">= 6"
      }
    },
+    "node_modules/classnames": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz",
+      "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw=="
+    },
    "node_modules/client-only": {
      "version": "0.0.1",
      "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
@@ -1245,6 +1252,18 @@
        "react": "^18.2.0"
      }
    },
+    "node_modules/react-dropdown": {
+      "version": "1.11.0",
+      "resolved": "https://registry.npmjs.org/react-dropdown/-/react-dropdown-1.11.0.tgz",
+      "integrity": "sha512-E2UWetRPxNdIhQahXw6b984ME7WmcgDj9AEAjrtS/oyLCFVo+2qkCfcS06C22JR0Zj+YLnygwv0Ozf6VKKDq7g==",
+      "dependencies": {
+        "classnames": "^2.2.3"
+      },
+      "peerDependencies": {
+        "react": "^0.14.7 || ^15.0.0-0 || ^16.0.0 || ^17.0.0 || ^18.0.0",
+        "react-dom": "^0.14.7 || ^15.0.0-0 || ^16.0.0 || ^17.0.0|| ^18.0.0"
+      }
+    },
    "node_modules/read-cache": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
@@ -1607,6 +1626,11 @@
        "node": ">=10.13.0"
      }
    },
+    "node_modules/wavesurfer.js": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-7.0.3.tgz",
+      "integrity": "sha512-gJ3P+Bd3Q4E8qETjjg0pneaVqm2J7jegG2Cc6vqEF5YDDKQ3m8sKsvVfgVhJkacKkO9jFAGDu58Hw4zLr7xD0A=="
+    },
    "node_modules/wrappy": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
--- a/package.json
+++ b/package.json
@@ -16,9 +16,11 @@
    "react": "^18.2.0",
    "react-dom": "^18.2.0",
    "sass": "^1.63.6",
+    "react-dropdown": "^1.11.0",
    "simple-peer": "^9.11.1",
    "supports-color": "^9.4.0",
-    "tailwindcss": "^3.3.2"
+    "tailwindcss": "^3.3.2",
+    "wavesurfer.js": "^7.0.3"
  },
  "main": "index.js",
  "repository": "https://github.com/Monadical-SAS/reflector-ui.git",
--- a/public/button.css
+++ b/public/button.css
@@ -45,7 +45,7 @@ button.block {
 input[type="button"][disabled],
 button[disabled] {
  border-color: #ccc;
-  background-color: #eee;
+  background: #b8b8b8 !important;
  cursor: not-allowed;
 }

--- a/yarn.lock
+++ b/yarn.lock
@@ -192,6 +192,11 @@ chokidar@^3.5.3, "chokidar@>=3.0.0 <4.0.0":
  optionalDependencies:
    fsevents "~2.3.2"

+classnames@^2.2.3:
+  version "2.3.2"
+  resolved "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz"
+  integrity sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==
+
 client-only@0.0.1:
  version "0.0.1"
  resolved "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz"
@@ -629,7 +634,7 @@ randombytes@^2.1.0:
  dependencies:
    safe-buffer "^5.1.0"

-react-dom@^18.2.0:
+"react-dom@^0.14.7 || ^15.0.0-0 || ^16.0.0 || ^17.0.0|| ^18.0.0", react-dom@^18.2.0:
  version "18.2.0"
  resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz"
  integrity sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==
@@ -637,7 +642,14 @@ react-dom@^18.2.0:
    loose-envify "^1.1.0"
    scheduler "^0.23.0"

-react@^18.2.0, "react@>= 16.8.0 || 17.x.x || ^18.0.0-0":
+react-dropdown@^1.11.0:
+  version "1.11.0"
+  resolved "https://registry.npmjs.org/react-dropdown/-/react-dropdown-1.11.0.tgz"
+  integrity sha512-E2UWetRPxNdIhQahXw6b984ME7WmcgDj9AEAjrtS/oyLCFVo+2qkCfcS06C22JR0Zj+YLnygwv0Ozf6VKKDq7g==
+  dependencies:
+    classnames "^2.2.3"
+
+"react@^0.14.7 || ^15.0.0-0 || ^16.0.0 || ^17.0.0 || ^18.0.0", react@^18.2.0, "react@>= 16.8.0 || 17.x.x || ^18.0.0-0":
  version "18.2.0"
  resolved "https://registry.npmjs.org/react/-/react-18.2.0.tgz"
  integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==
@@ -850,6 +862,11 @@ watchpack@2.4.0:
    glob-to-regexp "^0.4.1"
    graceful-fs "^4.1.2"

+wavesurfer.js@^7.0.3:
+  version "7.0.3"
+  resolved "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-7.0.3.tgz"
+  integrity sha512-gJ3P+Bd3Q4E8qETjjg0pneaVqm2J7jegG2Cc6vqEF5YDDKQ3m8sKsvVfgVhJkacKkO9jFAGDu58Hw4zLr7xD0A==
+
 wrappy@1:
  version "1.0.2"
  resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"