From 2cf61b191fe9f7b2162d07c35f88631938f39a47 Mon Sep 17 00:00:00 2001 From: Sara Date: Mon, 9 Oct 2023 15:52:11 +0200 Subject: [PATCH 1/5] fix waveform generation --- server/reflector/utils/audio_waveform.py | 17 ++++++++++------- server/reflector/views/transcripts.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/server/reflector/utils/audio_waveform.py b/server/reflector/utils/audio_waveform.py index f31e1748..7a6fdb3e 100644 --- a/server/reflector/utils/audio_waveform.py +++ b/server/reflector/utils/audio_waveform.py @@ -9,11 +9,12 @@ def get_audio_waveform(path: Path | str, segments_count: int = 256) -> list[int] path = path.as_posix() container = av.open(path) - stream = container.streams.get(audio=0)[0] + stream = container.streams.audio[0] duration = container.duration / av.time_base chunk_size_secs = duration / segments_count chunk_size = int(chunk_size_secs * stream.rate * stream.channels) + if chunk_size == 0: # there is not enough data to fill the chunks # so basically we use chunk_size of 1. @@ -22,7 +23,7 @@ def get_audio_waveform(path: Path | str, segments_count: int = 256) -> list[int] # 1.1 is a safety margin as it seems that pyav decode # does not always return the exact number of chunks # that we expect. - volumes = np.zeros(int(segments_count * 1.1), dtype=int) + volumes = np.zeros(int(segments_count * 1.1), dtype=float) current_chunk_idx = 0 current_chunk_size = 0 current_chunk_volume = 0 @@ -35,7 +36,6 @@ def get_audio_waveform(path: Path | str, segments_count: int = 256) -> list[int] count += len(data) frames += 1 samples += frame.samples - while len(data) > 0: datalen = len(data) @@ -53,13 +53,16 @@ def get_audio_waveform(path: Path | str, segments_count: int = 256) -> list[int] current_chunk_idx += 1 current_chunk_size = 0 current_chunk_volume = 0 - volumes = volumes[:current_chunk_idx] - # normalize the volumes 0-128 - volumes = volumes * 128 / volumes.max() + # number of decimals to use when rounding the peak value + digits = 2 + max_val = float(max(volumes)) + new_volumes = [] + for x in volumes: + new_volumes.append(round(x / max_val, digits)) - return volumes.astype("uint8").tolist() + return new_volumes if __name__ == "__main__": diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 410839d7..f7ab0f40 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -41,7 +41,7 @@ def generate_transcript_name(): class AudioWaveform(BaseModel): - data: list[int] + data: list[float] class TranscriptText(BaseModel): From f65813ec60412742ff2beb7de193951681969b52 Mon Sep 17 00:00:00 2001 From: Sara Date: Mon, 9 Oct 2023 15:53:21 +0200 Subject: [PATCH 2/5] recorder extract styles, use server waveform --- www/app/styles/recorder.js | 30 ++++++++++++++ www/app/transcripts/recorder.tsx | 68 ++++++++++++++------------------ 2 files changed, 60 insertions(+), 38 deletions(-) create mode 100644 www/app/styles/recorder.js diff --git a/www/app/styles/recorder.js b/www/app/styles/recorder.js new file mode 100644 index 00000000..dc9ace60 --- /dev/null +++ b/www/app/styles/recorder.js @@ -0,0 +1,30 @@ +export const waveSurferStyles = { + playerSettings: { + waveColor: "#777", + progressColor: "#222", + cursorColor: "OrangeRed", + }, + playerStyle: { + cursor: "pointer", + backgroundColor: "RGB(240 240 240)", + borderRadius: "15px", + }, + marker: ` + border-left: solid 1px orange; + padding: 0 2px 0 5px; + font-size: 0.7rem; + border-radius: 0 3px 3px 0; + + position: absolute; + width: 100px; + max-width: fit-content; + cursor: pointer; + background-color: white; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + transition: width 100ms linear; + z-index: 0; + `, + markerHover: { backgroundColor: "orange" }, +}; diff --git a/www/app/transcripts/recorder.tsx b/www/app/transcripts/recorder.tsx index ed86a73f..3bba6ef7 100644 --- a/www/app/transcripts/recorder.tsx +++ b/www/app/transcripts/recorder.tsx @@ -14,6 +14,7 @@ import { AudioWaveform } from "../api"; import AudioInputsDropdown from "./audioInputsDropdown"; import { Option } from "react-dropdown"; import { useError } from "../(errors)/errorContext"; +import { waveSurferStyles } from "../styles/recorder"; type RecorderProps = { setStream?: React.Dispatch>; @@ -94,20 +95,30 @@ export default function Recorder(props: RecorderProps) { }; }; + // Setup Shortcuts + useEffect(() => { + if (!record) return; + + return setupProjectorKeys(); + }, [record, deviceId]); + + // Waveform setup useEffect(() => { if (waveformRef.current) { + console.log(props.waveform); const _wavesurfer = WaveSurfer.create({ container: waveformRef.current, - waveColor: "#777", - progressColor: "#222", - cursorColor: "OrangeRed", + url: props.transcriptId + ? `${process.env.NEXT_PUBLIC_API_URL}/v1/transcripts/${props.transcriptId}/audio/mp3` + : undefined, + peaks: props.waveform?.data, + hideScrollbar: true, autoCenter: true, barWidth: 2, height: "auto", - url: props.transcriptId - ? `${process.env.NEXT_PUBLIC_API_URL}/v1/transcripts/${props.transcriptId}/audio/mp3` - : undefined, + + ...waveSurferStyles.player, }); if (!props.transcriptId) { @@ -115,10 +126,12 @@ export default function Recorder(props: RecorderProps) { _wshack.renderer.renderSingleCanvas = () => {}; } + // styling const wsWrapper = _wavesurfer.getWrapper(); - wsWrapper.style.cursor = "pointer"; - wsWrapper.style.backgroundColor = "RGB(240 240 240)"; - wsWrapper.style.borderRadius = "15px"; + wsWrapper.style.cursor = waveSurferStyles.playerStyle.cursor; + wsWrapper.style.backgroundColor = + waveSurferStyles.playerStyle.backgroundColor; + wsWrapper.style.borderRadius = waveSurferStyles.playerStyle.borderRadius; _wavesurfer.on("play", () => { setIsPlaying(true); @@ -131,9 +144,10 @@ export default function Recorder(props: RecorderProps) { setRecord(_wavesurfer.registerPlugin(RecordPlugin.create())); setWaveRegions(_wavesurfer.registerPlugin(CustomRegionsPlugin.create())); - if (props.transcriptId) _wavesurfer.toggleInteraction(true); + if (props.isPastMeeting) _wavesurfer.toggleInteraction(true); setWavesurfer(_wavesurfer); + return () => { _wavesurfer.destroy(); setIsRecording(false); @@ -152,35 +166,18 @@ export default function Recorder(props: RecorderProps) { if (!waveRegions) return; waveRegions.clearRegions(); + for (let topic of topicsRef.current) { const content = document.createElement("div"); - content.setAttribute( - "style", - ` - position: absolute; - border-left: solid 1px orange; - padding: 0 2px 0 5px; - font-size: 0.7rem; - width: 100px; - max-width: fit-content; - cursor: pointer; - background-color: white; - border-radius: 0 3px 3px 0; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - transition: width 100ms linear; - `, - ); + content.setAttribute("style", waveSurferStyles.marker); content.onmouseover = () => { - content.style.backgroundColor = "orange"; + content.style.backgroundColor = + waveSurferStyles.markerHover.backgroundColor; content.style.zIndex = "999"; content.style.width = "300px"; }; content.onmouseout = () => { - content.style.backgroundColor = "white"; - content.style.zIndex = "0"; - content.style.width = "100px"; + content.setAttribute("style", waveSurferStyles.marker); }; content.textContent = topic.title; @@ -198,12 +195,6 @@ export default function Recorder(props: RecorderProps) { } }; - useEffect(() => { - if (!record) return; - - return setupProjectorKeys(); - }, [record, deviceId]); - useEffect(() => { if (!record) return; @@ -214,6 +205,7 @@ export default function Recorder(props: RecorderProps) { link.setAttribute("href", record.getRecordedUrl()); link.setAttribute("download", "reflector-recording.webm"); link.style.visibility = "visible"; + renderMarkers(); }); }, [record]); From b66cfb5dacc77adeeda341a862226b9907323e68 Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 12 Oct 2023 14:36:43 +0100 Subject: [PATCH 3/5] Remove console.log --- www/app/transcripts/recorder.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/www/app/transcripts/recorder.tsx b/www/app/transcripts/recorder.tsx index 3bba6ef7..2ef6acac 100644 --- a/www/app/transcripts/recorder.tsx +++ b/www/app/transcripts/recorder.tsx @@ -105,7 +105,6 @@ export default function Recorder(props: RecorderProps) { // Waveform setup useEffect(() => { if (waveformRef.current) { - console.log(props.waveform); const _wavesurfer = WaveSurfer.create({ container: waveformRef.current, url: props.transcriptId From 006ab10b4439a755a9044b0953765e8aad890b73 Mon Sep 17 00:00:00 2001 From: Sara Date: Fri, 13 Oct 2023 10:50:46 +0200 Subject: [PATCH 4/5] use numpy in waveform calculation --- server/reflector/utils/audio_waveform.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/reflector/utils/audio_waveform.py b/server/reflector/utils/audio_waveform.py index 7a6fdb3e..8fd184a8 100644 --- a/server/reflector/utils/audio_waveform.py +++ b/server/reflector/utils/audio_waveform.py @@ -57,12 +57,9 @@ def get_audio_waveform(path: Path | str, segments_count: int = 256) -> list[int] # number of decimals to use when rounding the peak value digits = 2 - max_val = float(max(volumes)) - new_volumes = [] - for x in volumes: - new_volumes.append(round(x / max_val, digits)) + volumes = np.round(volumes / volumes.max(), digits) - return new_volumes + return volumes if __name__ == "__main__": From fb7a7602f396a818d59839cae02ddb76aa230473 Mon Sep 17 00:00:00 2001 From: Sara Date: Fri, 13 Oct 2023 11:34:22 +0200 Subject: [PATCH 5/5] waveform is a list --- server/reflector/utils/audio_waveform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/reflector/utils/audio_waveform.py b/server/reflector/utils/audio_waveform.py index 8fd184a8..d9f6b05c 100644 --- a/server/reflector/utils/audio_waveform.py +++ b/server/reflector/utils/audio_waveform.py @@ -59,7 +59,7 @@ def get_audio_waveform(path: Path | str, segments_count: int = 256) -> list[int] digits = 2 volumes = np.round(volumes / volumes.max(), digits) - return volumes + return volumes.tolist() if __name__ == "__main__":