From 0cf929dfa8c518de9717d77f7ab072c323e6b5da Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 20 Jul 2023 00:40:11 +0700 Subject: [PATCH] Full back end integration --- app/components/dashboard.js | 116 +++++++++--------------------- app/components/record.js | 21 +----- app/components/webrtc.js | 54 +++++++++----- app/{globals.css => globals.scss} | 5 +- app/layout.js | 2 +- app/page.js | 15 ++-- package-lock.json | 22 ++++++ package.json | 1 + yarn.lock | 18 ++++- 9 files changed, 120 insertions(+), 134 deletions(-) rename app/{globals.css => globals.scss} (90%) diff --git a/app/components/dashboard.js b/app/components/dashboard.js index 154dd076..198450c8 100644 --- a/app/components/dashboard.js +++ b/app/components/dashboard.js @@ -2,41 +2,21 @@ import { Mulberry32 } from "../utils.js"; import React, { useState, useEffect } from "react"; import AudioVisualizer from "./audioVisualizer.js"; -export function Dashboard(props) { +export function Dashboard({ + isRecording, + onRecord, + transcriptionText, + finalSummary, + topics, + stream, +}) { const [openIndex, setOpenIndex] = useState(null); const [liveTranscript, setLiveTranscript] = useState(""); - const [fakeTranscriptIndex, setFakeTranscriptIndex] = useState(0); - - const fakeTranscripts = [ - "This is the first transcript. We are discussing the current situation of our company. We are currently leading the market with a significant margin, and our future outlook is also very promising...", - "Here is the second transcript. We are now moving to our next topic, which is the progress in our ongoing projects. Most of them are on schedule and the quality of work is up to our standard...", - "This is the third transcript. It's about the financial status of our company. We are doing quite well financially. The revenue for this quarter is higher than expected...", - // add more fake transcripts as needed - ]; - - useEffect(() => { - // Randomly select a fake transcript - const selectedTranscript = - fakeTranscripts[Math.floor(Math.random() * fakeTranscripts.length)]; - // Split the selected transcript into characters - const characters = Array.from(selectedTranscript); - - let counter = 0; - let liveTranscriptCopy = ""; - let intervalId = setInterval(() => { - if (counter < characters.length) { - liveTranscriptCopy += characters[counter]; - setLiveTranscript(liveTranscriptCopy); - counter++; - } else { - clearInterval(intervalId); - } - }, 50); // delay of 100ms - - // Cleanup function to clear the interval when the component unmounts - return () => clearInterval(intervalId); - }, []); + topics = topics.map((topic, i) => { + topic["decibel"] = generateDecibelData(i + 1 + 333); // for looks only + return topic; + }); const generateDecibelData = (x) => { let data = []; @@ -58,50 +38,6 @@ export function Dashboard(props) { )); }; - // This is hardcoded data for proof of concept - const data = [ - { - timestamp: "00:00", - topic: "Meeting Introduction", - decibel: generateDecibelData(1), - transcript: - "This is the meeting introduction, we will be discussing several important topics today.", - }, - { - timestamp: "00:48", - topic: "Discussing Quarterly Revenue", - decibel: generateDecibelData(2), - transcript: - "We are discussing the quarterly revenue here, it appears our revenue has grown by 15% compared to the previous quarter.", - }, - { - timestamp: "01:35", - topic: "Annual Sales Review", - decibel: generateDecibelData(3), - transcript: - "Now we're reviewing the annual sales, there was a significant boost during the holiday season.", - }, - { - timestamp: "02:20", - topic: "Operational Costs Analysis", - decibel: generateDecibelData(4), - transcript: - "Moving on to the operational costs analysis, we have managed to reduce unnecessary expenses.", - }, - { - timestamp: "03:10", - topic: "Employee Performance", - decibel: generateDecibelData(5), - transcript: - "Let's talk about the employee performance, overall the team has done a great job.", - }, - /* { timestamp: '03:45', topic: 'New Marketing Strategies', decibel: generateDecibelData(6), transcript: "Our marketing team has proposed some new strategies that we'll discuss now." }, - { timestamp: '04:30', topic: 'Customer Feedback', decibel: generateDecibelData(7), transcript: "Let's go through some customer feedback that we've received." }, - { timestamp: '05:15', topic: 'Product Development', decibel: generateDecibelData(8), transcript: "Product development is going well and the new product line will be ready to launch next quarter." }, - { timestamp: '06:00', topic: 'Discussing Future Projects', decibel: generateDecibelData(9), transcript: "Now we are talking about the future projects, we have some exciting projects lined up." }, - { timestamp: '06:45', topic: 'Meeting Conclusion', decibel: generateDecibelData(10), transcript: "As we conclude the meeting, I want to thank everyone for their hard work and dedication." }, */ - ]; - return ( <>
@@ -114,7 +50,8 @@ export function Dashboard(props) {
Topic
- {data.map((item, index) => ( + + {topics.map((item, index) => (
{item.timestamp}
- {item.topic}{" "} + {item.title}{" "}
{openIndex === index && ( -
{item.transcript}
+
{item.description}
)}
))} +
Live
@@ -148,18 +86,28 @@ export function Dashboard(props) { {generateDecibelGraph(generateDecibelData())}
-
{props.transcriptionText}
+
+ {transcriptionText} +
- + + {finalSummary && ( +
+

Final Summary

+

Duration: {finalSummary.duration}

+

{finalSummary.summary}

+
+ )} +
Reflector © 2023 Monadical
diff --git a/app/components/record.js b/app/components/record.js index 03c9c4aa..a3d3e17f 100644 --- a/app/components/record.js +++ b/app/components/record.js @@ -1,21 +1,4 @@ export default function Record(props) { - let mediaRecorder = null; // mediaRecorder instance - - const startRecording = () => { - navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => { - mediaRecorder = new MediaRecorder(stream); - mediaRecorder.start(); - props.onRecord(true); - }); - }; - - const stopRecording = () => { - if (mediaRecorder) { - mediaRecorder.stop(); - props.onRecord(false); - } - }; - return (
@@ -25,11 +8,11 @@ export default function Record(props) {
{!props.isRecording ? ( - ) : ( - )} diff --git a/app/components/webrtc.js b/app/components/webrtc.js index f13e8356..88fd2b7b 100644 --- a/app/components/webrtc.js +++ b/app/components/webrtc.js @@ -3,16 +3,17 @@ import Peer from "simple-peer"; const WebRTC_SERVER_URL = "http://127.0.0.1:1250/offer"; -const useWebRTC = (stream) => { - const [data, setData] = useState(null); +const useWebRTC = (stream, setIsRecording) => { + const [data, setData] = useState({}); useEffect(() => { + if (!stream) { + return; + } + let peer = new Peer({ initiator: true, stream: stream }); peer.on("signal", (data) => { - // This is where you'd send the signal data to the server. - // The server would then send it back to other peers who would then - // use `peer.signal()` method to continue the connection negotiation. if ("sdp" in data) { fetch(WebRTC_SERVER_URL, { body: JSON.stringify({ @@ -24,13 +25,9 @@ const useWebRTC = (stream) => { }, method: "POST", }) - .then(function (response) { - return response.json(); - }) - .then(function (answer) { - return peer.signal(answer); - }) - .catch(function (e) { + .then((response) => response.json()) + .then((answer) => peer.signal(answer)) + .catch((e) => { alert(e); }); } @@ -41,17 +38,40 @@ const useWebRTC = (stream) => { }); peer.on("data", (data) => { - // Received data from the server. - console.log(data.toString()); const serverData = JSON.parse(data.toString()); - setData(serverData); + + switch (serverData.cmd) { + case "SHOW_TRANSCRIPTION": + setData((prevData) => ({ + ...prevData, + text: serverData.text, + })); + break; + case "UPDATE_TOPICS": + setData((prevData) => ({ + ...prevData, + topics: serverData.topics, + })); + break; + case "DISPLAY_FINAL_SUMMARY": + setData((prevData) => ({ + ...prevData, + finalSummary: { + duration: serverData.duration, + summary: serverData.summary, + }, + })); + setIsRecording(false); + break; + default: + console.error(`Unknown command ${serverData.cmd}`); + } }); - // Clean up return () => { peer.destroy(); }; - }, [stream]); + }, [stream, setIsRecording]); return data; }; diff --git a/app/globals.css b/app/globals.scss similarity index 90% rename from app/globals.css rename to app/globals.scss index 2d8b5900..abe75d80 100644 --- a/app/globals.css +++ b/app/globals.scss @@ -17,7 +17,7 @@ } body { - color: rgb(var(--foreground-rgb)); + color: black; background: linear-gradient( to bottom, transparent, @@ -28,8 +28,7 @@ body { font-family: "Roboto", sans-serif; } -.temp-transcription -{ +.temp-transcription { background: beige; border-radius: 5px; } diff --git a/app/layout.js b/app/layout.js index df1a0ac7..0e91a9e4 100644 --- a/app/layout.js +++ b/app/layout.js @@ -1,4 +1,4 @@ -import "./globals.css"; +import "./globals.scss"; import { Roboto } from "next/font/google"; import Head from "next/head"; diff --git a/app/page.js b/app/page.js index a0d9cd65..ea3fee00 100644 --- a/app/page.js +++ b/app/page.js @@ -1,5 +1,5 @@ "use client"; -import React, { useState, useEffect } from "react"; +import React, { useState } from "react"; import Record from "./components/record.js"; import { Dashboard } from "./components/dashboard.js"; import useWebRTC from "./components/webrtc.js"; @@ -8,10 +8,9 @@ import "../public/button.css"; const App = () => { const [isRecording, setIsRecording] = useState(false); const [splashScreen, setSplashScreen] = useState(true); + const [stream, setStream] = useState(null); const handleRecord = (recording) => { - console.log("handleRecord", recording); - setIsRecording(recording); setSplashScreen(false); @@ -26,13 +25,10 @@ const App = () => { tracks.forEach((track) => track.stop()); setStream(null); } - - setIsRecording(false); } }; - const [stream, setStream] = useState(null); - const serverData = useWebRTC(stream); + const serverData = useWebRTC(stream, setIsRecording); const text = serverData?.text ?? ""; return ( @@ -47,7 +43,10 @@ const App = () => { handleRecord(recording)} - transcriptionText={`[${serverData?.timestamp?.substring(2) ?? "??"}] ${text}`} + transcriptionText={text ?? "(No transcription text)"} + finalSummary={serverData.finalSummary} + topics={serverData.topics ?? []} + stream={stream} /> )}
diff --git a/package-lock.json b/package-lock.json index cb583783..e2b473d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "postcss": "8.4.25", "react": "^18.2.0", "react-dom": "^18.2.0", + "sass": "^1.63.6", "simple-peer": "^9.11.1", "supports-color": "^9.4.0", "tailwindcss": "^3.3.2" @@ -724,6 +725,11 @@ } ] }, + "node_modules/immutable": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.1.tgz", + "integrity": "sha512-lj9cnmB/kVS0QHsJnYKD1uo3o39nrbKxszjnqS9Fr6NB7bZzW45U6WSGBPKXDL/CvDKqDNPA4r3DoDQ8GTxo2A==" + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1337,6 +1343,22 @@ } ] }, + "node_modules/sass": { + "version": "1.63.6", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz", + "integrity": "sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw==", + "dependencies": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + }, + "bin": { + "sass": "sass.js" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/scheduler": { "version": "0.23.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz", diff --git a/package.json b/package.json index 228e4608..46ad0c7e 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "postcss": "8.4.25", "react": "^18.2.0", "react-dom": "^18.2.0", + "sass": "^1.63.6", "simple-peer": "^9.11.1", "supports-color": "^9.4.0", "tailwindcss": "^3.3.2" diff --git a/yarn.lock b/yarn.lock index 3ac29930..69869ae8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -177,7 +177,7 @@ caniuse-lite@^1.0.30001406, caniuse-lite@^1.0.30001464, caniuse-lite@^1.0.300015 resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001515.tgz" integrity sha512-eEFDwUOZbE24sb+Ecsx3+OvNETqjWIdabMy52oOkIgcUtAsQifjUG9q4U9dgTHJM2mfk4uEPxc0+xuFdJ629QA== -chokidar@^3.5.3: +chokidar@^3.5.3, "chokidar@>=3.0.0 <4.0.0": version "3.5.3" resolved "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz" integrity sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw== @@ -349,6 +349,11 @@ ieee754@^1.2.1: resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== +immutable@^4.0.0: + version "4.3.1" + resolved "https://registry.npmjs.org/immutable/-/immutable-4.3.1.tgz" + integrity sha512-lj9cnmB/kVS0QHsJnYKD1uo3o39nrbKxszjnqS9Fr6NB7bZzW45U6WSGBPKXDL/CvDKqDNPA4r3DoDQ8GTxo2A== + inflight@^1.0.4: version "1.0.6" resolved "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" @@ -688,6 +693,15 @@ safe-buffer@^5.1.0, safe-buffer@~5.2.0: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== +sass@^1.3.0, sass@^1.63.6: + version "1.63.6" + resolved "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz" + integrity sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw== + dependencies: + chokidar ">=3.0.0 <4.0.0" + immutable "^4.0.0" + source-map-js ">=0.6.2 <2.0.0" + scheduler@^0.23.0: version "0.23.0" resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz" @@ -708,7 +722,7 @@ simple-peer@^9.11.1: randombytes "^2.1.0" readable-stream "^3.6.0" -source-map-js@^1.0.2: +source-map-js@^1.0.2, "source-map-js@>=0.6.2 <2.0.0": version "1.0.2" resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz" integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==