From 7f7a13416d84c23687800efeb055adbedbb8e862 Mon Sep 17 00:00:00 2001 From: Jose B Date: Thu, 17 Aug 2023 14:35:36 -0500 Subject: [PATCH 1/5] useAudioDevice hook + permission mngmnt --- www/app/transcripts/new/page.js | 71 +++++++++++++++++++++------ www/app/transcripts/recorder.js | 43 ++++++---------- www/app/transcripts/useAudioDevice.js | 67 +++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 44 deletions(-) create mode 100644 www/app/transcripts/useAudioDevice.js diff --git a/www/app/transcripts/new/page.js b/www/app/transcripts/new/page.js index 60f4642b..65ebcb90 100644 --- a/www/app/transcripts/new/page.js +++ b/www/app/transcripts/new/page.js @@ -5,6 +5,7 @@ import { Dashboard } from "../dashboard"; import useWebRTC from "../useWebRTC"; import useTranscript from "../useTranscript"; import { useWebSockets } from "../useWebSockets"; +import useAudioDevice from "../useAudioDevice"; import "../../styles/button.css"; const App = () => { @@ -24,6 +25,13 @@ const App = () => { const transcript = useTranscript(); const webRTC = useWebRTC(stream, transcript.response?.id); const webSockets = useWebSockets(transcript.response?.id); + const { + loading, + permissionOk, + audioDevices, + requestPermission, + getAudioStream, + } = useAudioDevice(); return (
@@ -32,23 +40,54 @@ const App = () => {

Capture The Signal, Not The Noise

- { - webRTC?.peer?.send(JSON.stringify({ cmd: "STOP" })); - setStream(null); - }} - /> + {permissionOk ? ( + <> + { + webRTC?.peer?.send(JSON.stringify({ cmd: "STOP" })); + setStream(null); + }} + getAudioStream={getAudioStream} + audioDevices={audioDevices} + /> -
- - + + + ) : ( + <> +
+

+ Audio Permissions +

+ {loading ? ( +

+ Checking permission... +

+ ) : ( + <> +

+ Reflector needs access to your microphone to work. +
+ Please grant permission to continue. +

+ + + )} +
+ + )} ); }; diff --git a/www/app/transcripts/recorder.js b/www/app/transcripts/recorder.js index 41284236..134373bb 100644 --- a/www/app/transcripts/recorder.js +++ b/www/app/transcripts/recorder.js @@ -15,24 +15,11 @@ const AudioInputsDropdown = (props) => { const [ddOptions, setDdOptions] = useState([]); useEffect(() => { - const init = async () => { - // Request permission to use audio inputs - await navigator.mediaDevices - .getUserMedia({ audio: true }) - .then((stream) => stream.getTracks().forEach((t) => t.stop())); - - const devices = await navigator.mediaDevices.enumerateDevices(); - const audioDevices = devices - .filter((d) => d.kind === "audioinput" && d.deviceId != "") - .map((d) => ({ value: d.deviceId, label: d.label })); - - if (audioDevices.length < 1) return console.log("no audio input devices"); - - setDdOptions(audioDevices); - props.setDeviceId(audioDevices[0].value); - }; - init(); - }, []); + setDdOptions(props.audioDevices); + props.setDeviceId( + props.audioDevices.length > 0 ? props.audioDevices[0].value : null, + ); + }, [props.audioDevices]); const handleDropdownChange = (e) => { props.setDeviceId(e.value); @@ -131,16 +118,12 @@ export default function Recorder(props) { setIsRecording(false); document.getElementById("play-btn").disabled = false; } else { - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - deviceId, - noiseSuppression: false, - echoCancellation: false, - }, - }); - await record.startRecording(stream); + const stream = await props.getAudioStream(deviceId); props.setStream(stream); - setIsRecording(true); + if (stream) { + await record.startRecording(stream); + setIsRecording(true); + } } }; @@ -158,7 +141,11 @@ export default function Recorder(props) { return (
- +   )} diff --git a/www/app/transcripts/useAudioDevice.ts b/www/app/transcripts/useAudioDevice.ts index da1dbe49..0b430adc 100644 --- a/www/app/transcripts/useAudioDevice.ts +++ b/www/app/transcripts/useAudioDevice.ts @@ -1,22 +1,57 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import { Option } from "react-dropdown"; +const MIC_QUERY = { name: "microphone" as PermissionName }; + const useAudioDevice = () => { - const [permissionOk, setPermissionOk] = useState(false); + const [permissionOk, setPermissionOk] = useState(false); + const [permissionDenied, setPermissionDenied] = useState(false); const [audioDevices, setAudioDevices] = useState([]); const [loading, setLoading] = useState(true); + useEffect(() => { + checkPermission(); + }, []); + + useEffect(() => { + if (permissionOk) { + updateDevices(); + } + }, [permissionOk]); + + const checkPermission = (): void => { + navigator.permissions + .query(MIC_QUERY) + .then((permissionStatus) => { + setPermissionOk(permissionStatus.state === "granted"); + setPermissionDenied(permissionStatus.state === "denied"); + permissionStatus.onchange = () => { + setPermissionOk(permissionStatus.state === "granted"); + setPermissionDenied(permissionStatus.state === "denied"); + }; + }) + .catch(() => { + setPermissionOk(false); + setPermissionDenied(false); + }) + .finally(() => { + setLoading(false); + }); + }; + const requestPermission = () => { navigator.mediaDevices .getUserMedia({ audio: true, }) - .then(() => { + .then((stream) => { + if (!navigator.userAgent.includes("Firefox")) + stream.getTracks().forEach((track) => track.stop()); setPermissionOk(true); - updateDevices(); }) .catch(() => { + setPermissionDenied(true); setPermissionOk(false); }) .finally(() => { @@ -56,11 +91,12 @@ const useAudioDevice = () => { }; return { + loading, permissionOk, + permissionDenied, audioDevices, getAudioStream, requestPermission, - loading, }; }; From 8b7e80d7cbee27d07b60261f87a430d7606f1a10 Mon Sep 17 00:00:00 2001 From: Jose B Date: Tue, 22 Aug 2023 20:06:47 -0500 Subject: [PATCH 4/5] update tsconfig --- www/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/www/tsconfig.json b/www/tsconfig.json index 9c9b16c2..0e1b89ae 100644 --- a/www/tsconfig.json +++ b/www/tsconfig.json @@ -18,7 +18,8 @@ "name": "next" } ], - "strictNullChecks": true + "strictNullChecks": true, + "downlevelIteration": true }, "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"] From b5c43616491cdcaf040225270c902af7dc329b37 Mon Sep 17 00:00:00 2001 From: Jose B Date: Wed, 23 Aug 2023 15:05:48 -0500 Subject: [PATCH 5/5] add typing --- www/app/transcripts/new/page.tsx | 3 ++- www/app/transcripts/recorder.tsx | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/www/app/transcripts/new/page.tsx b/www/app/transcripts/new/page.tsx index 06908e89..4eb5f47c 100644 --- a/www/app/transcripts/new/page.tsx +++ b/www/app/transcripts/new/page.tsx @@ -48,9 +48,10 @@ const App = () => { webRTC?.peer?.send(JSON.stringify({ cmd: "STOP" })); setStream(null); }} + topics={webSockets.topics} getAudioStream={getAudioStream} audioDevices={audioDevices} - topics={webSockets.topics} + useActiveTopic={useActiveTopic} /> >; + onStop: () => void; + topics: Topic[]; + getAudioStream: (deviceId: string | null) => Promise; + audioDevices: Option[]; + useActiveTopic: [ + Topic | null, + React.Dispatch>, + ]; +}; + +export default function Recorder(props: RecorderProps) { const waveformRef = useRef(null); const [wavesurfer, setWavesurfer] = useState(null); const [record, setRecord] = useState(null);