Merge pull request #144 from Monadical-SAS/jose/markers

Waveform markers + sync accordion topics + TS

www/app/lib/custom-plugins/record.ts

@@ -1,7 +1,21 @@
-// Override the startRecording method so we can pass the desired stream
+// Source code: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts
-// Checkout: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/record.ts
+/**
+ * Record audio from the microphone, render a waveform and download the audio.
+ */
 
-import RecordPlugin from "wavesurfer.js/dist/plugins/record";
+import BasePlugin, {
+  type BasePluginEvents,
+} from "wavesurfer.js/dist/base-plugin";
+
+export type RecordPluginOptions = {
+  mimeType?: MediaRecorderOptions["mimeType"];
+  audioBitsPerSecond?: MediaRecorderOptions["audioBitsPerSecond"];
+};
+
+export type RecordPluginEvents = BasePluginEvents & {
+  startRecording: [];
+  stopRecording: [];
+};
 
 const MIME_TYPES = [
   "audio/webm",
@@ -13,11 +27,44 @@ const MIME_TYPES = [
 const findSupportedMimeType = () =>
   MIME_TYPES.find((mimeType) => MediaRecorder.isTypeSupported(mimeType));
 
-class CustomRecordPlugin extends RecordPlugin {
+class RecordPlugin extends BasePlugin<RecordPluginEvents, RecordPluginOptions> {
-  static create(options) {
+  private mediaRecorder: MediaRecorder | null = null;
-    return new CustomRecordPlugin(options || {});
+  private recordedUrl = "";
+  private savedCursorWidth = 1;
+  private savedInteractive = true;
+
+  public static create(options?: RecordPluginOptions) {
+    return new RecordPlugin(options || {});
   }
-  render(stream) {
+
+  private preventInteraction() {
+    if (this.wavesurfer) {
+      this.savedCursorWidth = this.wavesurfer.options.cursorWidth || 1;
+      this.savedInteractive = this.wavesurfer.options.interact || true;
+      this.wavesurfer.options.cursorWidth = 0;
+      this.wavesurfer.options.interact = false;
+    }
+  }
+
+  private restoreInteraction() {
+    if (this.wavesurfer) {
+      this.wavesurfer.options.cursorWidth = this.savedCursorWidth;
+      this.wavesurfer.options.interact = this.savedInteractive;
+    }
+  }
+
+  onInit() {
+    this.preventInteraction();
+  }
+
+  private loadBlob(data: Blob[], type: string) {
+    const blob = new Blob(data, { type });
+    this.recordedUrl = URL.createObjectURL(blob);
+    this.restoreInteraction();
+    this.wavesurfer?.load(this.recordedUrl);
+  }
+
+  render(stream: MediaStream): () => void {
     if (!this.wavesurfer) return () => undefined;
 
     const container = this.wavesurfer.getWrapper();
@@ -36,11 +83,12 @@ class CustomRecordPlugin extends RecordPlugin {
     const bufferLength = analyser.frequencyBinCount;
     const dataArray = new Uint8Array(bufferLength);
 
-    let animationId, previousTimeStamp;
+    let animationId: number, previousTimeStamp: number;
+    const DATA_SIZE = 128.0;
     const BUFFER_SIZE = 2 ** 8;
-    const dataBuffer = new Array(BUFFER_SIZE).fill(canvas.height);
+    const dataBuffer = new Array(BUFFER_SIZE).fill(DATA_SIZE);
 
-    const drawWaveform = (timeStamp) => {
+    const drawWaveform = (timeStamp: number) => {
       if (!canvasCtx) return;
 
       analyser.getByteTimeDomainData(dataArray);
@@ -64,9 +112,9 @@ class CustomRecordPlugin extends RecordPlugin {
       let x = 0;
 
       for (let i = 0; i < dataBuffer.length; i++) {
-        const valueNormalized = dataBuffer[i] / canvas.height;
+        const y = (canvas.height * dataBuffer[i]) / (2 * DATA_SIZE);
-        const y = (valueNormalized * canvas.height) / 2;
+        const sliceHeight =
-        const sliceHeight = canvas.height + 1 - y * 2;
+          ((1 - canvas.height) * dataBuffer[i]) / DATA_SIZE + canvas.height;
 
         canvasCtx.fillRect(x, y, (sliceWidth * 2) / 3, sliceHeight);
         x += sliceWidth;
@@ -75,7 +123,7 @@ class CustomRecordPlugin extends RecordPlugin {
       animationId = requestAnimationFrame(drawWaveform);
     };
 
-    drawWaveform();
+    drawWaveform(0);
 
     return () => {
       if (animationId) {
@@ -94,7 +142,17 @@ class CustomRecordPlugin extends RecordPlugin {
       canvas?.remove();
     };
   }
-  startRecording(stream) {
+
+  private cleanUp() {
+    this.stopRecording();
+    this.wavesurfer?.empty();
+    if (this.recordedUrl) {
+      URL.revokeObjectURL(this.recordedUrl);
+      this.recordedUrl = "";
+    }
+  }
+
+  public async startRecording(stream: MediaStream) {
     this.preventInteraction();
     this.cleanUp();
 
@@ -103,7 +161,7 @@ class CustomRecordPlugin extends RecordPlugin {
       mimeType: this.options.mimeType || findSupportedMimeType(),
       audioBitsPerSecond: this.options.audioBitsPerSecond,
     });
-    const recordedChunks = [];
+    const recordedChunks: Blob[] = [];
 
     mediaRecorder.addEventListener("dataavailable", (event) => {
       if (event.data.size > 0) {
@@ -123,6 +181,25 @@ class CustomRecordPlugin extends RecordPlugin {
 
     this.mediaRecorder = mediaRecorder;
   }
+
+  public isRecording(): boolean {
+    return this.mediaRecorder?.state === "recording";
+  }
+
+  public stopRecording() {
+    if (this.isRecording()) {
+      this.mediaRecorder?.stop();
+    }
+  }
+
+  public getRecordedUrl(): string {
+    return this.recordedUrl;
+  }
+
+  public destroy() {
+    super.destroy();
+    this.cleanUp();
+  }
 }
 
-export default CustomRecordPlugin;
+export default RecordPlugin;

www/app/lib/custom-plugins/regions.ts

@@ -0,0 +1,18 @@
+// Source code: https://github.com/katspaugh/wavesurfer.js/blob/fa2bcfe/src/plugins/regions.ts
+
+import RegionsPlugin, {
+  RegionsPluginOptions,
+} from "wavesurfer.js/dist/plugins/regions";
+
+class CustomRegionsPlugin extends RegionsPlugin {
+  public static create(options?: RegionsPluginOptions) {
+    return new CustomRegionsPlugin(options);
+  }
+
+  constructor(options?: RegionsPluginOptions) {
+    super(options);
+    this["avoidOverlapping"] = () => {};
+  }
+}
+
+export default CustomRegionsPlugin;

www/app/transcripts/dashboard.tsx

@@ -16,6 +16,10 @@ type DashboardProps = {
   finalSummary: FinalSummaryType;
   topics: Topic[];
   disconnected: boolean;
+  useActiveTopic: [
+    Topic | null,
+    React.Dispatch<React.SetStateAction<Topic | null>>,
+  ];
 };
 
 export function Dashboard({
@@ -23,8 +27,9 @@ export function Dashboard({
   finalSummary,
   topics,
   disconnected,
+  useActiveTopic,
 }: DashboardProps) {
-  const [openIndex, setOpenIndex] = useState<number | null>(null);
+  const [activeTopic, setActiveTopic] = useActiveTopic;
   const [autoscrollEnabled, setAutoscrollEnabled] = useState<boolean>(true);
 
   useEffect(() => {
@@ -51,7 +56,7 @@ export function Dashboard({
 
   return (
     <>
-      <div className="relative h-[60svh] w-3/4 flex flex-col">
+      <div className="relative h-[64svh] w-3/4 flex flex-col">
         <div className="text-center pb-1 pt-4">
           <h1 className="text-2xl font-bold text-blue-500">Meeting Notes</h1>
         </div>
@@ -75,18 +80,24 @@ export function Dashboard({
             <div key={index} className="border-b-2 py-2 hover:bg-[#8ec5fc30]">
               <div
                 className="flex justify-between items-center cursor-pointer px-4"
-                onClick={() => setOpenIndex(openIndex === index ? null : index)}
+                onClick={() =>
+                  setActiveTopic(activeTopic?.id == item.id ? null : item)
+                }
               >
                 <div className="w-1/4">{formatTime(item.timestamp)}</div>
                 <div className="w-3/4 flex justify-between items-center">
                   {item.title}
                   <FontAwesomeIcon
                     className={`transform transition-transform duration-200`}
-                    icon={openIndex === index ? faChevronDown : faChevronRight}
+                    icon={
+                      activeTopic?.id == item.id
+                        ? faChevronDown
+                        : faChevronRight
+                    }
                   />
                 </div>
               </div>
-              {openIndex === index && (
+              {activeTopic?.id == item.id && (
                 <div className="p-2 mt-2 -mb-2 bg-slate-50 rounded">
                   {item.transcript}
                 </div>

www/app/transcripts/new/page.tsx

@@ -6,11 +6,13 @@ import useWebRTC from "../useWebRTC";
 import useTranscript from "../useTranscript";
 import { useWebSockets } from "../useWebSockets";
 import "../../styles/button.css";
+import { Topic } from "../webSocketTypes";
 import getApi from "../../lib/getApi";
 
 const App = () => {
   const [stream, setStream] = useState<MediaStream | null>(null);
   const [disconnected, setDisconnected] = useState<boolean>(false);
+  const useActiveTopic = useState<Topic | null>(null);
 
   useEffect(() => {
     if (process.env.NEXT_PUBLIC_ENV === "development") {
@@ -35,6 +37,8 @@ const App = () => {
           webRTC?.peer?.send(JSON.stringify({ cmd: "STOP" }));
           setStream(null);
         }}
+        topics={webSockets.topics}
+        useActiveTopic={useActiveTopic}
       />
 
       <hr />
@@ -44,6 +48,7 @@ const App = () => {
         finalSummary={webSockets.finalSummary}
         topics={webSockets.topics}
         disconnected={disconnected}
+        useActiveTopic={useActiveTopic}
       />
     </>
   );

www/app/transcripts/recorder.tsx

@@ -1,18 +1,22 @@
 import React, { useRef, useEffect, useState } from "react";
 
 import WaveSurfer from "wavesurfer.js";
+import RecordPlugin from "../lib/custom-plugins/record";
+import CustomRegionsPlugin from "../lib/custom-plugins/regions";
 
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { faDownload } from "@fortawesome/free-solid-svg-icons";
 
-import Dropdown from "react-dropdown";
+import Dropdown, { Option } from "react-dropdown";
 import "react-dropdown/style.css";
 
-import CustomRecordPlugin from "../lib/CustomRecordPlugin";
 import { formatTime } from "../lib/time";
 
-const AudioInputsDropdown = (props) => {
+const AudioInputsDropdown: React.FC<{
-  const [ddOptions, setDdOptions] = useState([]);
+  setDeviceId: React.Dispatch<React.SetStateAction<string | null>>;
+  disabled: boolean;
+}> = (props) => {
+  const [ddOptions, setDdOptions] = useState<Array<Option>>([]);
 
   useEffect(() => {
     const init = async () => {
@@ -34,8 +38,8 @@ const AudioInputsDropdown = (props) => {
     init();
   }, []);
 
-  const handleDropdownChange = (e) => {
+  const handleDropdownChange = (option: Option) => {
-    props.setDeviceId(e.value);
+    props.setDeviceId(option.value);
   };
 
   return (
@@ -48,19 +52,27 @@ const AudioInputsDropdown = (props) => {
   );
 };
 
-export default function Recorder(props) {
+export default function Recorder(props: any) {
-  const waveformRef = useRef();
+  const waveformRef = useRef<HTMLDivElement>(null);
-  const [wavesurfer, setWavesurfer] = useState(null);
+  const [wavesurfer, setWavesurfer] = useState<WaveSurfer | null>(null);
-  const [record, setRecord] = useState(null);
+  const [record, setRecord] = useState<RecordPlugin | null>(null);
-  const [isRecording, setIsRecording] = useState(false);
+  const [isRecording, setIsRecording] = useState<boolean>(false);
-  const [isPlaying, setIsPlaying] = useState(false);
+  const [isPlaying, setIsPlaying] = useState<boolean>(false);
-  const [deviceId, setDeviceId] = useState(null);
+  const [deviceId, setDeviceId] = useState<string | null>(null);
-  const [currentTime, setCurrentTime] = useState(0);
+  const [currentTime, setCurrentTime] = useState<number>(0);
-  const [timeInterval, setTimeInterval] = useState(null);
+  const [timeInterval, setTimeInterval] = useState<number | null>(null);
-  const [duration, setDuration] = useState(0);
+  const [duration, setDuration] = useState<number>(0);
+  const [waveRegions, setWaveRegions] = useState<CustomRegionsPlugin | null>(
+    null,
+  );
+
+  const [activeTopic, setActiveTopic] = props.useActiveTopic;
+
+  const topicsRef = useRef(props.topics);
 
   useEffect(() => {
-    document.getElementById("play-btn").disabled = true;
+    const playBtn = document.getElementById("play-btn");
+    if (playBtn) playBtn.setAttribute("disabled", "true");
 
     if (waveformRef.current) {
       const _wavesurfer = WaveSurfer.create({
@@ -71,6 +83,7 @@ export default function Recorder(props) {
         hideScrollbar: true,
         autoCenter: true,
         barWidth: 2,
+        height: 90,
       });
       const wsWrapper = _wavesurfer.getWrapper();
       wsWrapper.style.cursor = "pointer";
@@ -85,36 +98,97 @@ export default function Recorder(props) {
       });
       _wavesurfer.on("timeupdate", setCurrentTime);
 
-      setRecord(_wavesurfer.registerPlugin(CustomRecordPlugin.create()));
+      setRecord(_wavesurfer.registerPlugin(RecordPlugin.create()));
+      setWaveRegions(_wavesurfer.registerPlugin(CustomRegionsPlugin.create()));
+
       setWavesurfer(_wavesurfer);
       return () => {
         _wavesurfer.destroy();
         setIsRecording(false);
         setIsPlaying(false);
+        setCurrentTime(0);
       };
     }
   }, []);
 
+  useEffect(() => {
+    topicsRef.current = props.topics;
+    if (!isRecording) renderMarkers();
+  }, [props.topics]);
+
+  const renderMarkers = () => {
+    if (!waveRegions) return;
+
+    waveRegions.clearRegions();
+    for (let topic of topicsRef.current) {
+      const content = document.createElement("div");
+      content.setAttribute(
+        "style",
+        `
+        position: absolute;
+        border-left: solid 1px orange;
+        padding: 0 2px 0 5px;
+        font-size: 0.7rem;
+        width: 100px;
+        max-width: fit-content;
+        cursor: pointer;
+        background-color: white;
+        border-radius: 0 3px 3px 0;
+        white-space: nowrap;
+        overflow: hidden;
+        text-overflow: ellipsis;
+        transition: width 100ms linear;
+      `,
+      );
+      content.onmouseover = () => {
+        content.style.backgroundColor = "orange";
+        content.style.zIndex = "999";
+        content.style.width = "300px";
+      };
+      content.onmouseout = () => {
+        content.style.backgroundColor = "white";
+        content.style.zIndex = "0";
+        content.style.width = "100px";
+      };
+      content.textContent = topic.title;
+
+      const region = waveRegions.addRegion({
+        start: topic.timestamp,
+        content,
+        color: "f00",
+        drag: false,
+      });
+      region.on("click", (e) => {
+        e.stopPropagation();
+        setActiveTopic(topic);
+        wavesurfer?.setTime(region.start);
+      });
+    }
+  };
+
   useEffect(() => {
     if (record) {
       return record.on("stopRecording", () => {
         const link = document.getElementById("download-recording");
-        link.href = record.getRecordedUrl();
+        if (!link) return;
-        link.download = "reflector-recording.webm";
+
+        link.setAttribute("href", record.getRecordedUrl());
+        link.setAttribute("download", "reflector-recording.webm");
         link.style.visibility = "visible";
+        renderMarkers();
       });
     }
   }, [record]);
 
   useEffect(() => {
     if (isRecording) {
-      const interval = setInterval(() => {
+      const interval = window.setInterval(() => {
         setCurrentTime((prev) => prev + 1);
       }, 1000);
       setTimeInterval(interval);
       return () => clearInterval(interval);
     } else {
-      clearInterval(timeInterval);
+      clearInterval(timeInterval as number);
       setCurrentTime((prev) => {
         setDuration(prev);
         return 0;
@@ -122,6 +196,12 @@ export default function Recorder(props) {
     }
   }, [isRecording]);
 
+  useEffect(() => {
+    if (activeTopic) {
+      wavesurfer?.setTime(activeTopic.timestamp);
+    }
+  }, [activeTopic]);
+
   const handleRecClick = async () => {
     if (!record) return console.log("no record");
 
@@ -129,11 +209,12 @@ export default function Recorder(props) {
       props.onStop();
       record.stopRecording();
       setIsRecording(false);
-      document.getElementById("play-btn").disabled = false;
+      const playBtn = document.getElementById("play-btn");
+      if (playBtn) playBtn.removeAttribute("disabled");
     } else {
       const stream = await navigator.mediaDevices.getUserMedia({
         audio: {
-          deviceId,
+          deviceId: deviceId as string,
           noiseSuppression: false,
           echoCancellation: false,
         },
@@ -141,6 +222,7 @@ export default function Recorder(props) {
       await record.startRecording(stream);
       props.setStream(stream);
       setIsRecording(true);
+      waveRegions?.clearRegions();
     }
   };
 

www/tsconfig.json

@@ -18,7 +18,8 @@
         "name": "next"
       }
     ],
-    "strictNullChecks": true
+    "strictNullChecks": true,
+    "downlevelIteration": true
   },
   "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"],
   "exclude": ["node_modules"]