From 0cf929dfa8c518de9717d77f7ab072c323e6b5da Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 20 Jul 2023 00:40:11 +0700 Subject: [PATCH 1/6] Full back end integration --- app/components/dashboard.js | 116 +++++++++--------------------- app/components/record.js | 21 +----- app/components/webrtc.js | 54 +++++++++----- app/{globals.css => globals.scss} | 5 +- app/layout.js | 2 +- app/page.js | 15 ++-- package-lock.json | 22 ++++++ package.json | 1 + yarn.lock | 18 ++++- 9 files changed, 120 insertions(+), 134 deletions(-) rename app/{globals.css => globals.scss} (90%) diff --git a/app/components/dashboard.js b/app/components/dashboard.js index 154dd076..198450c8 100644 --- a/app/components/dashboard.js +++ b/app/components/dashboard.js @@ -2,41 +2,21 @@ import { Mulberry32 } from "../utils.js"; import React, { useState, useEffect } from "react"; import AudioVisualizer from "./audioVisualizer.js"; -export function Dashboard(props) { +export function Dashboard({ + isRecording, + onRecord, + transcriptionText, + finalSummary, + topics, + stream, +}) { const [openIndex, setOpenIndex] = useState(null); const [liveTranscript, setLiveTranscript] = useState(""); - const [fakeTranscriptIndex, setFakeTranscriptIndex] = useState(0); - - const fakeTranscripts = [ - "This is the first transcript. We are discussing the current situation of our company. We are currently leading the market with a significant margin, and our future outlook is also very promising...", - "Here is the second transcript. We are now moving to our next topic, which is the progress in our ongoing projects. Most of them are on schedule and the quality of work is up to our standard...", - "This is the third transcript. It's about the financial status of our company. We are doing quite well financially. The revenue for this quarter is higher than expected...", - // add more fake transcripts as needed - ]; - - useEffect(() => { - // Randomly select a fake transcript - const selectedTranscript = - fakeTranscripts[Math.floor(Math.random() * fakeTranscripts.length)]; - // Split the selected transcript into characters - const characters = Array.from(selectedTranscript); - - let counter = 0; - let liveTranscriptCopy = ""; - let intervalId = setInterval(() => { - if (counter < characters.length) { - liveTranscriptCopy += characters[counter]; - setLiveTranscript(liveTranscriptCopy); - counter++; - } else { - clearInterval(intervalId); - } - }, 50); // delay of 100ms - - // Cleanup function to clear the interval when the component unmounts - return () => clearInterval(intervalId); - }, []); + topics = topics.map((topic, i) => { + topic["decibel"] = generateDecibelData(i + 1 + 333); // for looks only + return topic; + }); const generateDecibelData = (x) => { let data = []; @@ -58,50 +38,6 @@ export function Dashboard(props) { )); }; - // This is hardcoded data for proof of concept - const data = [ - { - timestamp: "00:00", - topic: "Meeting Introduction", - decibel: generateDecibelData(1), - transcript: - "This is the meeting introduction, we will be discussing several important topics today.", - }, - { - timestamp: "00:48", - topic: "Discussing Quarterly Revenue", - decibel: generateDecibelData(2), - transcript: - "We are discussing the quarterly revenue here, it appears our revenue has grown by 15% compared to the previous quarter.", - }, - { - timestamp: "01:35", - topic: "Annual Sales Review", - decibel: generateDecibelData(3), - transcript: - "Now we're reviewing the annual sales, there was a significant boost during the holiday season.", - }, - { - timestamp: "02:20", - topic: "Operational Costs Analysis", - decibel: generateDecibelData(4), - transcript: - "Moving on to the operational costs analysis, we have managed to reduce unnecessary expenses.", - }, - { - timestamp: "03:10", - topic: "Employee Performance", - decibel: generateDecibelData(5), - transcript: - "Let's talk about the employee performance, overall the team has done a great job.", - }, - /* { timestamp: '03:45', topic: 'New Marketing Strategies', decibel: generateDecibelData(6), transcript: "Our marketing team has proposed some new strategies that we'll discuss now." }, - { timestamp: '04:30', topic: 'Customer Feedback', decibel: generateDecibelData(7), transcript: "Let's go through some customer feedback that we've received." }, - { timestamp: '05:15', topic: 'Product Development', decibel: generateDecibelData(8), transcript: "Product development is going well and the new product line will be ready to launch next quarter." }, - { timestamp: '06:00', topic: 'Discussing Future Projects', decibel: generateDecibelData(9), transcript: "Now we are talking about the future projects, we have some exciting projects lined up." }, - { timestamp: '06:45', topic: 'Meeting Conclusion', decibel: generateDecibelData(10), transcript: "As we conclude the meeting, I want to thank everyone for their hard work and dedication." }, */ - ]; - return ( <>
@@ -114,7 +50,8 @@ export function Dashboard(props) {
Topic
- {data.map((item, index) => ( + + {topics.map((item, index) => (
{item.timestamp}
- {item.topic}{" "} + {item.title}{" "}
{openIndex === index && ( -
{item.transcript}
+
{item.description}
)}
))} +
Live
@@ -148,18 +86,28 @@ export function Dashboard(props) { {generateDecibelGraph(generateDecibelData())}
-
{props.transcriptionText}
+
+ {transcriptionText} +
- + + {finalSummary && ( +
+

Final Summary

+

Duration: {finalSummary.duration}

+

{finalSummary.summary}

+
+ )} +
Reflector © 2023 Monadical
diff --git a/app/components/record.js b/app/components/record.js index 03c9c4aa..a3d3e17f 100644 --- a/app/components/record.js +++ b/app/components/record.js @@ -1,21 +1,4 @@ export default function Record(props) { - let mediaRecorder = null; // mediaRecorder instance - - const startRecording = () => { - navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { - mediaRecorder = new MediaRecorder(stream); - mediaRecorder.start(); - props.onRecord(true); - }); - }; - - const stopRecording = () => { - if (mediaRecorder) { - mediaRecorder.stop(); - props.onRecord(false); - } - }; - return (
@@ -25,11 +8,11 @@ export default function Record(props) {
{!props.isRecording ? ( - ) : ( - )} diff --git a/app/components/webrtc.js b/app/components/webrtc.js index f13e8356..88fd2b7b 100644 --- a/app/components/webrtc.js +++ b/app/components/webrtc.js @@ -3,16 +3,17 @@ import Peer from "simple-peer"; const WebRTC_SERVER_URL = "http://127.0.0.1:1250/offer"; -const useWebRTC = (stream) => { - const [data, setData] = useState(null); +const useWebRTC = (stream, setIsRecording) => { + const [data, setData] = useState({}); useEffect(() => { + if (!stream) { + return; + } + let peer = new Peer({ initiator: true, stream: stream }); peer.on("signal", (data) => { - // This is where you'd send the signal data to the server. - // The server would then send it back to other peers who would then - // use `peer.signal()` method to continue the connection negotiation. if ("sdp" in data) { fetch(WebRTC_SERVER_URL, { body: JSON.stringify({ @@ -24,13 +25,9 @@ const useWebRTC = (stream) => { }, method: "POST", }) - .then(function (response) { - return response.json(); - }) - .then(function (answer) { - return peer.signal(answer); - }) - .catch(function (e) { + .then((response) => response.json()) + .then((answer) => peer.signal(answer)) + .catch((e) => { alert(e); }); } @@ -41,17 +38,40 @@ const useWebRTC = (stream) => { }); peer.on("data", (data) => { - // Received data from the server. - console.log(data.toString()); const serverData = JSON.parse(data.toString()); - setData(serverData); + + switch (serverData.cmd) { + case "SHOW_TRANSCRIPTION": + setData((prevData) => ({ + ...prevData, + text: serverData.text, + })); + break; + case "UPDATE_TOPICS": + setData((prevData) => ({ + ...prevData, + topics: serverData.topics, + })); + break; + case "DISPLAY_FINAL_SUMMARY": + setData((prevData) => ({ + ...prevData, + finalSummary: { + duration: serverData.duration, + summary: serverData.summary, + }, + })); + setIsRecording(false); + break; + default: + console.error(`Unknown command ${serverData.cmd}`); + } }); - // Clean up return () => { peer.destroy(); }; - }, [stream]); + }, [stream, setIsRecording]); return data; }; diff --git a/app/globals.css b/app/globals.scss similarity index 90% rename from app/globals.css rename to app/globals.scss index 2d8b5900..abe75d80 100644 --- a/app/globals.css +++ b/app/globals.scss @@ -17,7 +17,7 @@ } body { - color: rgb(var(--foreground-rgb)); + color: black; background: linear-gradient( to bottom, transparent, @@ -28,8 +28,7 @@ body { font-family: "Roboto", sans-serif; } -.temp-transcription -{ +.temp-transcription { background: beige; border-radius: 5px; } diff --git a/app/layout.js b/app/layout.js index df1a0ac7..0e91a9e4 100644 --- a/app/layout.js +++ b/app/layout.js @@ -1,4 +1,4 @@ -import "./globals.css"; +import "./globals.scss"; import { Roboto } from "next/font/google"; import Head from "next/head"; diff --git a/app/page.js b/app/page.js index a0d9cd65..ea3fee00 100644 --- a/app/page.js +++ b/app/page.js @@ -1,5 +1,5 @@ "use client"; -import React, { useState, useEffect } from "react"; +import React, { useState } from "react"; import Record from "./components/record.js"; import { Dashboard } from "./components/dashboard.js"; import useWebRTC from "./components/webrtc.js"; @@ -8,10 +8,9 @@ import "../public/button.css"; const App = () => { const [isRecording, setIsRecording] = useState(false); const [splashScreen, setSplashScreen] = useState(true); + const [stream, setStream] = useState(null); const handleRecord = (recording) => { - console.log("handleRecord", recording); - setIsRecording(recording); setSplashScreen(false); @@ -26,13 +25,10 @@ const App = () => { tracks.forEach((track) => track.stop()); setStream(null); } - - setIsRecording(false); } }; - const [stream, setStream] = useState(null); - const serverData = useWebRTC(stream); + const serverData = useWebRTC(stream, setIsRecording); const text = serverData?.text ?? ""; return ( @@ -47,7 +43,10 @@ const App = () => { handleRecord(recording)} - transcriptionText={`[${serverData?.timestamp?.substring(2) ?? "??"}] ${text}`} + transcriptionText={text ?? "(No transcription text)"} + finalSummary={serverData.finalSummary} + topics={serverData.topics ?? []} + stream={stream} /> )}
diff --git a/package-lock.json b/package-lock.json index cb583783..e2b473d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "postcss": "8.4.25", "react": "^18.2.0", "react-dom": "^18.2.0", + "sass": "^1.63.6", "simple-peer": "^9.11.1", "supports-color": "^9.4.0", "tailwindcss": "^3.3.2" @@ -724,6 +725,11 @@ } ] }, + "node_modules/immutable": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.1.tgz", + "integrity": "sha512-lj9cnmB/kVS0QHsJnYKD1uo3o39nrbKxszjnqS9Fr6NB7bZzW45U6WSGBPKXDL/CvDKqDNPA4r3DoDQ8GTxo2A==" + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1337,6 +1343,22 @@ } ] }, + "node_modules/sass": { + "version": "1.63.6", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz", + "integrity": "sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw==", + "dependencies": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + }, + "bin": { + "sass": "sass.js" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/scheduler": { "version": "0.23.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz", diff --git a/package.json b/package.json index 228e4608..46ad0c7e 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "postcss": "8.4.25", "react": "^18.2.0", "react-dom": "^18.2.0", + "sass": "^1.63.6", "simple-peer": "^9.11.1", "supports-color": "^9.4.0", "tailwindcss": "^3.3.2" diff --git a/yarn.lock b/yarn.lock index 3ac29930..69869ae8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -177,7 +177,7 @@ caniuse-lite@^1.0.30001406, caniuse-lite@^1.0.30001464, caniuse-lite@^1.0.300015 resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001515.tgz" integrity sha512-eEFDwUOZbE24sb+Ecsx3+OvNETqjWIdabMy52oOkIgcUtAsQifjUG9q4U9dgTHJM2mfk4uEPxc0+xuFdJ629QA== -chokidar@^3.5.3: +chokidar@^3.5.3, "chokidar@>=3.0.0 <4.0.0": version "3.5.3" resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== @@ -349,6 +349,11 @@ ieee754@^1.2.1: resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== +immutable@^4.0.0: + version "4.3.1" + resolved "https://registry.npmjs.org/immutable/-/immutable-4.3.1.tgz" + integrity sha512-lj9cnmB/kVS0QHsJnYKD1uo3o39nrbKxszjnqS9Fr6NB7bZzW45U6WSGBPKXDL/CvDKqDNPA4r3DoDQ8GTxo2A== + inflight@^1.0.4: version "1.0.6" resolved "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" @@ -688,6 +693,15 @@ safe-buffer@^5.1.0, safe-buffer@~5.2.0: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== +sass@^1.3.0, sass@^1.63.6: + version "1.63.6" + resolved "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz" + integrity sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw== + dependencies: + chokidar ">=3.0.0 <4.0.0" + immutable "^4.0.0" + source-map-js ">=0.6.2 <2.0.0" + scheduler@^0.23.0: version "0.23.0" resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz" @@ -708,7 +722,7 @@ simple-peer@^9.11.1: randombytes "^2.1.0" readable-stream "^3.6.0" -source-map-js@^1.0.2: +source-map-js@^1.0.2, "source-map-js@>=0.6.2 <2.0.0": version "1.0.2" resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz" integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== From 9163b5c1e5e2e7ccdec6a7f021e3abb60853b387 Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 20 Jul 2023 00:44:19 +0700 Subject: [PATCH 2/6] getting rid of legacy code --- app/page.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/page.js b/app/page.js index ea3fee00..b104eb03 100644 --- a/app/page.js +++ b/app/page.js @@ -29,7 +29,6 @@ const App = () => { }; const serverData = useWebRTC(stream, setIsRecording); - const text = serverData?.text ?? ""; return (
@@ -43,7 +42,7 @@ const App = () => { handleRecord(recording)} - transcriptionText={text ?? "(No transcription text)"} + transcriptionText={serverData.text ?? "(No transcription text)"} finalSummary={serverData.finalSummary} topics={serverData.topics ?? []} stream={stream} From 971a173a9b2d9ff291da209ab78bc0ff2945d8e2 Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 20 Jul 2023 12:53:24 +0700 Subject: [PATCH 3/6] Fixed uninitialized topic variable related to "fake" decibel graph which has been removed --- app/components/dashboard.js | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/app/components/dashboard.js b/app/components/dashboard.js index 198450c8..32c91e2e 100644 --- a/app/components/dashboard.js +++ b/app/components/dashboard.js @@ -13,31 +13,6 @@ export function Dashboard({ const [openIndex, setOpenIndex] = useState(null); const [liveTranscript, setLiveTranscript] = useState(""); - topics = topics.map((topic, i) => { - topic["decibel"] = generateDecibelData(i + 1 + 333); // for looks only - return topic; - }); - - const generateDecibelData = (x) => { - let data = []; - let random = Mulberry32(123456789 + x); - for (let i = 0; i < 50; i++) { - data.push(Math.floor(random() * 30) + 10); // generate random values between 10 and 40 - } - return data; - }; - const generateDecibelGraph = (decibelData) => { - return decibelData.map((decibel, i) => ( -
-   -
- )); - }; - return ( <>
@@ -69,7 +44,6 @@ export function Dashboard({
- {generateDecibelGraph(item.decibel)}
{openIndex === index && ( @@ -83,7 +57,6 @@ export function Dashboard({
Live
Transcript
- {generateDecibelGraph(generateDecibelData())}
From 71e5df5dac5db1efd0780e83b3edd1bbff04b95c Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 20 Jul 2023 15:54:28 +0700 Subject: [PATCH 4/6] Stop command --- app/components/webrtc.js | 5 ++++- app/page.js | 9 +++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/components/webrtc.js b/app/components/webrtc.js index 88fd2b7b..c81f2336 100644 --- a/app/components/webrtc.js +++ b/app/components/webrtc.js @@ -4,7 +4,9 @@ import Peer from "simple-peer"; const WebRTC_SERVER_URL = "http://127.0.0.1:1250/offer"; const useWebRTC = (stream, setIsRecording) => { - const [data, setData] = useState({}); + const [data, setData] = useState({ + peer: null, + }); useEffect(() => { if (!stream) { @@ -35,6 +37,7 @@ const useWebRTC = (stream, setIsRecording) => { peer.on("connect", () => { console.log("WebRTC connected"); + setData(prevData => ({ ...prevData, peer: peer })); }); peer.on("data", (data) => { diff --git a/app/page.js b/app/page.js index b104eb03..1aaf9b6c 100644 --- a/app/page.js +++ b/app/page.js @@ -19,15 +19,12 @@ const App = () => { .getUserMedia({ audio: true }) .then(setStream) .catch((err) => console.error(err)); - } else if (!recording) { - if (stream) { - const tracks = stream.getTracks(); - tracks.forEach((track) => track.stop()); - setStream(null); - } + } else if (!recording && serverData.peer) { + serverData.peer.send(JSON.stringify({ cmd: 'STOP' })); } }; + const serverData = useWebRTC(stream, setIsRecording); return ( From 959b7f4ca459e54f19615a57517417ba10d15fef Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 20 Jul 2023 18:52:46 +0700 Subject: [PATCH 5/6] Removed dark mode --- app/globals.scss | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app/globals.scss b/app/globals.scss index abe75d80..b16c3c87 100644 --- a/app/globals.scss +++ b/app/globals.scss @@ -8,16 +8,7 @@ --background-end-rgb: 255, 255, 255; } -@media (prefers-color-scheme: dark) { - :root { - --foreground-rgb: 255, 255, 255; - --background-start-rgb: 0, 0, 0; - --background-end-rgb: 0, 0, 0; - } -} - body { - color: black; background: linear-gradient( to bottom, transparent, From 154132e028e78852f73131e9297fe2d1bed19ed2 Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 20 Jul 2023 22:23:21 +0700 Subject: [PATCH 6/6] Changed the topic description to transcript and added console.log for webrtc incoming messages --- app/components/dashboard.js | 2 +- app/components/webrtc.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/components/dashboard.js b/app/components/dashboard.js index 32c91e2e..f5164d5f 100644 --- a/app/components/dashboard.js +++ b/app/components/dashboard.js @@ -47,7 +47,7 @@ export function Dashboard({
{openIndex === index && ( -
{item.description}
+
{item.transcript}
)} ))} diff --git a/app/components/webrtc.js b/app/components/webrtc.js index 88fd2b7b..73a17931 100644 --- a/app/components/webrtc.js +++ b/app/components/webrtc.js @@ -39,6 +39,7 @@ const useWebRTC = (stream, setIsRecording) => { peer.on("data", (data) => { const serverData = JSON.parse(data.toString()); + console.log(serverData); switch (serverData.cmd) { case "SHOW_TRANSCRIPTION":