From 0cdd7037fbf4bb93dcf2d77cf9567d041d2ce71f Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Wed, 16 Aug 2023 14:03:25 +0530 Subject: [PATCH 01/31] wrap JSONFormer around LLM --- server/gpu/modal/reflector_llm.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/server/gpu/modal/reflector_llm.py b/server/gpu/modal/reflector_llm.py index bf6f4cf5..315ff785 100644 --- a/server/gpu/modal/reflector_llm.py +++ b/server/gpu/modal/reflector_llm.py @@ -10,7 +10,7 @@ from modal import Image, method, Stub, asgi_app, Secret # LLM LLM_MODEL: str = "lmsys/vicuna-13b-v1.5" -LLM_LOW_CPU_MEM_USAGE: bool = False +LLM_LOW_CPU_MEM_USAGE: bool = True LLM_TORCH_DTYPE: str = "bfloat16" LLM_MAX_NEW_TOKENS: int = 300 @@ -49,6 +49,8 @@ llm_image = ( "torch", "sentencepiece", "protobuf", + "jsonformer==0.12.0", + "accelerate==0.21.0", "einops==0.6.1", "hf-transfer~=0.1", "huggingface_hub==0.16.4", @@ -81,6 +83,7 @@ class LLM: # generation configuration print("Instance llm generation config") + # JSONFormer doesn't yet support generation configs, but keeping for future usage model.config.max_new_tokens = LLM_MAX_NEW_TOKENS gen_cfg = GenerationConfig.from_model_config(model.config) gen_cfg.max_new_tokens = LLM_MAX_NEW_TOKENS @@ -97,6 +100,13 @@ class LLM: self.model = model self.tokenizer = tokenizer self.gen_cfg = gen_cfg + self.json_schema = { + "type": "object", + "properties": { + "title": {"type": "string"}, + "summary": {"type": "string"}, + }, + } def __exit__(self, *args): print("Exit llm") @@ -109,16 +119,17 @@ class LLM: @method() def generate(self, prompt: str): print(f"Generate {prompt=}") - # tokenize prompt - input_ids = self.tokenizer.encode(prompt, return_tensors="pt").to( - self.model.device - ) - output = self.model.generate(input_ids, generation_config=self.gen_cfg) + import jsonformer + import json - # decode output - response = self.tokenizer.decode(output[0].cpu(), skip_special_tokens=True) + jsonformer_llm = jsonformer.Jsonformer(model=self.model, + tokenizer=self.tokenizer, + json_schema=self.json_schema, + prompt=prompt, + max_string_token_length=self.gen_cfg.max_new_tokens) + response = jsonformer_llm() print(f"Generated {response=}") - return {"text": response} + return {"text": json.dumps(response)} # ------------------------------------------------------------------- From 976c0ab9a8c926e89f6430b0f4fcedb1c181f7d7 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Wed, 16 Aug 2023 14:07:29 +0530 Subject: [PATCH 02/31] update prompt --- server/reflector/processors/transcript_topic_detector.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/reflector/processors/transcript_topic_detector.py b/server/reflector/processors/transcript_topic_detector.py index b626e8a2..6e926771 100644 --- a/server/reflector/processors/transcript_topic_detector.py +++ b/server/reflector/processors/transcript_topic_detector.py @@ -14,10 +14,11 @@ class TranscriptTopicDetectorProcessor(Processor): PROMPT = """ ### Human: - Create a JSON object as response.The JSON object must have 2 fields: - i) title and ii) summary.For the title field,generate a short title - for the given text. For the summary field, summarize the given text - in three sentences. + Generate information based on the given schema: + + For the title field, generate a short title for the given text. + For the summary field, summarize the given text in a maximum of + three sentences. {input_text} From ebbf47b8956b341221c1b7e72f3033f8bd6ae260 Mon Sep 17 00:00:00 2001 From: Koper Date: Wed, 16 Aug 2023 17:55:47 +0700 Subject: [PATCH 03/31] Migrate to typescript --- www/app/{layout.js => layout.tsx} | 8 +- .../CustomRecordPlugin.js | 0 www/app/lib/{random.js => random.tsx} | 6 +- www/app/lib/{time.js => time.tsx} | 2 +- www/app/{page.js => page.tsx} | 2 +- .../{dashboard.js => dashboard.tsx} | 73 ++++----- www/app/transcripts/disconnectedIndicator.tsx | 13 ++ www/app/transcripts/finalSummary.tsx | 12 ++ www/app/transcripts/liveTranscription.tsx | 11 ++ www/app/transcripts/new/{page.js => page.tsx} | 5 +- www/app/transcripts/recorder.js | 2 +- www/app/transcripts/scrollToBottom.tsx | 23 +++ .../{useTranscript.js => useTranscript.tsx} | 20 ++- .../{useWebRTC.js => useWebRTC.tsx} | 30 ++-- .../{useWebSockets.js => useWebSockets.tsx} | 24 ++- www/app/transcripts/webSocketTypes.tsx | 19 +++ www/next.config.js | 2 +- www/package.json | 2 +- www/pages/api/sentry-example-api.js | 5 + www/pages/sentry-example-page.js | 87 +++++++++++ ...ient.config.js => sentry.client.config.ts} | 0 ...y.edge.config.js => sentry.edge.config.ts} | 0 ...rver.config.js => sentry.server.config.ts} | 0 www/yarn.lock | 138 +++++++++--------- 24 files changed, 329 insertions(+), 155 deletions(-) rename www/app/{layout.js => layout.tsx} (94%) rename www/app/{transcripts => lib}/CustomRecordPlugin.js (100%) rename www/app/lib/{random.js => random.tsx} (68%) rename www/app/lib/{time.js => time.tsx} (83%) rename www/app/{page.js => page.tsx} (60%) rename www/app/transcripts/{dashboard.js => dashboard.tsx} (59%) create mode 100644 www/app/transcripts/disconnectedIndicator.tsx create mode 100644 www/app/transcripts/finalSummary.tsx create mode 100644 www/app/transcripts/liveTranscription.tsx rename www/app/transcripts/new/{page.js => page.tsx} (91%) create mode 100644 www/app/transcripts/scrollToBottom.tsx rename www/app/transcripts/{useTranscript.js => useTranscript.tsx} (66%) rename www/app/transcripts/{useWebRTC.js => useWebRTC.tsx} (63%) rename www/app/transcripts/{useWebSockets.js => useWebSockets.tsx} (68%) create mode 100644 www/app/transcripts/webSocketTypes.tsx create mode 100644 www/pages/api/sentry-example-api.js create mode 100644 www/pages/sentry-example-page.js rename www/{sentry.client.config.js => sentry.client.config.ts} (100%) rename www/{sentry.edge.config.js => sentry.edge.config.ts} (100%) rename www/{sentry.server.config.js => sentry.server.config.ts} (100%) diff --git a/www/app/layout.js b/www/app/layout.tsx similarity index 94% rename from www/app/layout.js rename to www/app/layout.tsx index 824a8e16..99733717 100644 --- a/www/app/layout.js +++ b/www/app/layout.tsx @@ -1,11 +1,10 @@ import "./styles/globals.scss"; import { Roboto } from "next/font/google"; - -import Head from "next/head"; +import { Metadata } from "next"; const roboto = Roboto({ subsets: ["latin"], weight: "400" }); -export const metadata = { +export const metadata: Metadata = { title: { template: "%s – Reflector", default: "Reflector - AI-Powered Meeting Transcriptions by Monadical", @@ -52,9 +51,6 @@ export const metadata = { export default function RootLayout({ children }) { return ( - - Test - {children} diff --git a/www/app/transcripts/CustomRecordPlugin.js b/www/app/lib/CustomRecordPlugin.js similarity index 100% rename from www/app/transcripts/CustomRecordPlugin.js rename to www/app/lib/CustomRecordPlugin.js diff --git a/www/app/lib/random.js b/www/app/lib/random.tsx similarity index 68% rename from www/app/lib/random.js rename to www/app/lib/random.tsx index 37c4dee7..eebe0212 100644 --- a/www/app/lib/random.js +++ b/www/app/lib/random.tsx @@ -1,15 +1,15 @@ -export function getRandomNumber(min, max) { +export function getRandomNumber(min: number, max: number): number { return Math.floor(Math.random() * (max - min + 1)) + min; } -export function SeededRand(seed) { +export function SeededRand(seed: number): number { seed ^= seed << 13; seed ^= seed >> 17; seed ^= seed << 5; return seed / 2 ** 32; } -export function Mulberry32(seed) { +export function Mulberry32(seed: number) { return function () { var t = (seed += 0x6d2b79f5); t = Math.imul(t ^ (t >>> 15), t | 1); diff --git a/www/app/lib/time.js b/www/app/lib/time.tsx similarity index 83% rename from www/app/lib/time.js rename to www/app/lib/time.tsx index a6204ade..6d8a4e76 100644 --- a/www/app/lib/time.js +++ b/www/app/lib/time.tsx @@ -1,4 +1,4 @@ -export const formatTime = (seconds) => { +export const formatTime = (seconds: number): string => { let hours = Math.floor(seconds / 3600); let minutes = Math.floor((seconds % 3600) / 60); let secs = Math.floor(seconds % 60); diff --git a/www/app/page.js b/www/app/page.tsx similarity index 60% rename from www/app/page.js rename to www/app/page.tsx index d2835cf1..aff9fc3a 100644 --- a/www/app/page.js +++ b/www/app/page.tsx @@ -1,4 +1,4 @@ import { redirect } from "next/navigation"; -export default async function Index({ params }) { +export default async function Index() { redirect("/transcripts/new"); } diff --git a/www/app/transcripts/dashboard.js b/www/app/transcripts/dashboard.tsx similarity index 59% rename from www/app/transcripts/dashboard.js rename to www/app/transcripts/dashboard.tsx index 94bae58c..b426c670 100644 --- a/www/app/transcripts/dashboard.js +++ b/www/app/transcripts/dashboard.tsx @@ -3,17 +3,29 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faChevronRight, faChevronDown, - faLinkSlash, } from "@fortawesome/free-solid-svg-icons"; +import { formatTime } from "../lib/time"; +import ScrollToBottom from "./scrollToBottom"; +import FinalSummary from "./finalSummary"; +import DisconnectedIndicator from "./disconnectedIndicator"; +import LiveTrancription from "./liveTranscription"; +import { TopicType, FinalSummaryType } from "./webSocketTypes"; + +type DashboardProps = { + transcriptionText: string; + finalSummary: FinalSummaryType; + topics: TopicType[]; + disconnected: boolean; +}; export function Dashboard({ transcriptionText, finalSummary, topics, disconnected, -}) { - const [openIndex, setOpenIndex] = useState(null); - const [autoscrollEnabled, setAutoscrollEnabled] = useState(true); +}: DashboardProps) { + const [openIndex, setOpenIndex] = useState(null); + const [autoscrollEnabled, setAutoscrollEnabled] = useState(true); useEffect(() => { if (autoscrollEnabled) scrollToBottom(); @@ -21,7 +33,10 @@ export function Dashboard({ const scrollToBottom = () => { const topicsDiv = document.getElementById("topics-div"); - topicsDiv.scrollTop = topicsDiv.scrollHeight; + + if (!topicsDiv) + console.error("Could not find topics div to scroll to bottom"); + else topicsDiv.scrollTop = topicsDiv.scrollHeight; }; const handleScroll = (e) => { @@ -34,18 +49,6 @@ export function Dashboard({ } }; - const formatTime = (seconds) => { - let hours = Math.floor(seconds / 3600); - let minutes = Math.floor((seconds % 3600) / 60); - let secs = Math.floor(seconds % 60); - - let timeString = `${hours > 0 ? hours + ":" : ""}${minutes - .toString() - .padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; - - return timeString; - }; - return ( <>
@@ -57,16 +60,12 @@ export function Dashboard({
Topic
-
- ⬇ -
+ +
- {finalSummary && ( -
-

Final Summary

-

{finalSummary.summary}

-
- )} + {finalSummary && }
- {disconnected && ( -
-
- - Disconnected -
-
- )} + {disconnected && } -
-  {transcriptionText}  -
+ ); } diff --git a/www/app/transcripts/disconnectedIndicator.tsx b/www/app/transcripts/disconnectedIndicator.tsx new file mode 100644 index 00000000..275a54f5 --- /dev/null +++ b/www/app/transcripts/disconnectedIndicator.tsx @@ -0,0 +1,13 @@ +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faLinkSlash } from "@fortawesome/free-solid-svg-icons"; + +export default function DisconnectedIndicator() { + return ( +
+
+ + Disconnected +
+
+ ); +} diff --git a/www/app/transcripts/finalSummary.tsx b/www/app/transcripts/finalSummary.tsx new file mode 100644 index 00000000..4036b2ba --- /dev/null +++ b/www/app/transcripts/finalSummary.tsx @@ -0,0 +1,12 @@ +type FinalSummaryProps = { + text: string; +}; + +export default function FinalSummary(props: FinalSummaryProps) { + return ( +
+

Final Summary

+

{props.text}

+
+ ); +} diff --git a/www/app/transcripts/liveTranscription.tsx b/www/app/transcripts/liveTranscription.tsx new file mode 100644 index 00000000..6241cdb2 --- /dev/null +++ b/www/app/transcripts/liveTranscription.tsx @@ -0,0 +1,11 @@ +type LiveTranscriptionProps = { + text: string; +}; + +export default function LiveTrancription(props: LiveTranscriptionProps) { + return ( +
+  {props.text}  +
+ ); +} diff --git a/www/app/transcripts/new/page.js b/www/app/transcripts/new/page.tsx similarity index 91% rename from www/app/transcripts/new/page.js rename to www/app/transcripts/new/page.tsx index 60f4642b..80bbe9fe 100644 --- a/www/app/transcripts/new/page.js +++ b/www/app/transcripts/new/page.tsx @@ -8,8 +8,8 @@ import { useWebSockets } from "../useWebSockets"; import "../../styles/button.css"; const App = () => { - const [stream, setStream] = useState(null); - const [disconnected, setDisconnected] = useState(false); + const [stream, setStream] = useState(null); + const [disconnected, setDisconnected] = useState(false); useEffect(() => { if (process.env.NEXT_PUBLIC_ENV === "development") { @@ -46,7 +46,6 @@ const App = () => { transcriptionText={webSockets.transcriptText} finalSummary={webSockets.finalSummary} topics={webSockets.topics} - stream={stream} disconnected={disconnected} /> diff --git a/www/app/transcripts/recorder.js b/www/app/transcripts/recorder.js index 41284236..5eb06c82 100644 --- a/www/app/transcripts/recorder.js +++ b/www/app/transcripts/recorder.js @@ -8,7 +8,7 @@ import { faDownload } from "@fortawesome/free-solid-svg-icons"; import Dropdown from "react-dropdown"; import "react-dropdown/style.css"; -import CustomRecordPlugin from "./CustomRecordPlugin"; +import CustomRecordPlugin from "../lib/CustomRecordPlugin"; import { formatTime } from "../lib/time"; const AudioInputsDropdown = (props) => { diff --git a/www/app/transcripts/scrollToBottom.tsx b/www/app/transcripts/scrollToBottom.tsx new file mode 100644 index 00000000..9c267a66 --- /dev/null +++ b/www/app/transcripts/scrollToBottom.tsx @@ -0,0 +1,23 @@ +type ScrollToBottomProps = { + visible: boolean; + hasFinalSummary: boolean; + handleScrollBottom: () => void; +}; + +export default function ScrollToBottom(props: ScrollToBottomProps) { + return ( +
{ + props.handleScrollBottom(); + return false; + }} + > + ⬇ +
+ ); +} diff --git a/www/app/transcripts/useTranscript.js b/www/app/transcripts/useTranscript.tsx similarity index 66% rename from www/app/transcripts/useTranscript.js rename to www/app/transcripts/useTranscript.tsx index e846d626..ddb77bed 100644 --- a/www/app/transcripts/useTranscript.js +++ b/www/app/transcripts/useTranscript.tsx @@ -1,11 +1,19 @@ import { useEffect, useState } from "react"; -import { DefaultApi } from "../api/apis/DefaultApi"; +import { DefaultApi, V1TranscriptsCreateRequest } from "../api/apis/DefaultApi"; import { Configuration } from "../api/runtime"; +import { GetTranscript } from "../api"; -const useTranscript = () => { - const [response, setResponse] = useState(null); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); +type UseTranscriptReturnType = { + response: GetTranscript | null; + loading: boolean; + error: string | null; + createTranscript: () => void; +}; + +const useTranscript = (): UseTranscriptReturnType => { + const [response, setResponse] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); const apiConfiguration = new Configuration({ basePath: process.env.NEXT_PUBLIC_API_URL, @@ -14,7 +22,7 @@ const useTranscript = () => { const createTranscript = () => { setLoading(true); - const requestParameters = { + const requestParameters: V1TranscriptsCreateRequest = { createTranscript: { name: "Weekly All-Hands", // Hardcoded for now }, diff --git a/www/app/transcripts/useWebRTC.js b/www/app/transcripts/useWebRTC.tsx similarity index 63% rename from www/app/transcripts/useWebRTC.js rename to www/app/transcripts/useWebRTC.tsx index d34b0696..ea622bb2 100644 --- a/www/app/transcripts/useWebRTC.js +++ b/www/app/transcripts/useWebRTC.tsx @@ -1,12 +1,16 @@ import { useEffect, useState } from "react"; import Peer from "simple-peer"; -import { DefaultApi } from "../api/apis/DefaultApi"; +import { + DefaultApi, + V1TranscriptRecordWebrtcRequest, +} from "../api/apis/DefaultApi"; import { Configuration } from "../api/runtime"; -const useWebRTC = (stream, transcriptId) => { - const [data, setData] = useState({ - peer: null, - }); +const useWebRTC = ( + stream: MediaStream | null, + transcriptId: string | null, +): Peer => { + const [peer, setPeer] = useState(null); useEffect(() => { if (!stream || !transcriptId) { @@ -18,11 +22,11 @@ const useWebRTC = (stream, transcriptId) => { }); const api = new DefaultApi(apiConfiguration); - let peer = new Peer({ initiator: true, stream: stream }); + let p: Peer = new Peer({ initiator: true, stream: stream }); - peer.on("signal", (data) => { + p.on("signal", (data: any) => { if ("sdp" in data) { - const requestParameters = { + const requestParameters: V1TranscriptRecordWebrtcRequest = { transcriptId: transcriptId, rtcOffer: { sdp: data.sdp, @@ -33,7 +37,7 @@ const useWebRTC = (stream, transcriptId) => { api .v1TranscriptRecordWebrtc(requestParameters) .then((answer) => { - peer.signal(answer); + p.signal(answer); }) .catch((err) => { console.error("WebRTC signaling error:", err); @@ -41,17 +45,17 @@ const useWebRTC = (stream, transcriptId) => { } }); - peer.on("connect", () => { + p.on("connect", () => { console.log("WebRTC connected"); - setData((prevData) => ({ ...prevData, peer: peer })); + setPeer(p); }); return () => { - peer.destroy(); + p.destroy(); }; }, [stream, transcriptId]); - return data; + return peer; }; export default useWebRTC; diff --git a/www/app/transcripts/useWebSockets.js b/www/app/transcripts/useWebSockets.tsx similarity index 68% rename from www/app/transcripts/useWebSockets.js rename to www/app/transcripts/useWebSockets.tsx index 8b64790d..5e872ccb 100644 --- a/www/app/transcripts/useWebSockets.js +++ b/www/app/transcripts/useWebSockets.tsx @@ -1,10 +1,22 @@ import { useEffect, useState } from "react"; +import { TopicType, FinalSummaryType, StatusType } from "./webSocketTypes"; -export const useWebSockets = (transcriptId) => { - const [transcriptText, setTranscriptText] = useState(""); - const [topics, setTopics] = useState([]); - const [finalSummary, setFinalSummary] = useState(""); - const [status, setStatus] = useState("disconnected"); +type UseWebSocketsReturnType = { + transcriptText: string; + topics: TopicType[]; + finalSummary: FinalSummaryType; + status: StatusType; +}; + +export const useWebSockets = ( + transcriptId: string | null, +): UseWebSocketsReturnType => { + const [transcriptText, setTranscriptText] = useState(""); + const [topics, setTopics] = useState([]); + const [finalSummary, setFinalSummary] = useState({ + summary: "", + }); + const [status, setStatus] = useState({ value: "disconnected" }); useEffect(() => { if (!transcriptId) return; @@ -40,7 +52,7 @@ export const useWebSockets = (transcriptId) => { break; case "STATUS": - setStatus(message.data.status); + setStatus(message.data); break; default: diff --git a/www/app/transcripts/webSocketTypes.tsx b/www/app/transcripts/webSocketTypes.tsx new file mode 100644 index 00000000..6ae6cac6 --- /dev/null +++ b/www/app/transcripts/webSocketTypes.tsx @@ -0,0 +1,19 @@ +export type TopicType = { + timestamp: number; + title: string; + transcript: string; + summary: string; + id: string; +}; + +export type TranscriptType = { + text: string; +}; + +export type FinalSummaryType = { + summary: string; +}; + +export type StatusType = { + value: string; +}; diff --git a/www/next.config.js b/www/next.config.js index 610b5be8..e5be889f 100644 --- a/www/next.config.js +++ b/www/next.config.js @@ -5,7 +5,7 @@ const nextConfig = { module.exports = nextConfig; -// Sentry content below +// Injected content via Sentry wizard below const { withSentryConfig } = require("@sentry/nextjs"); diff --git a/www/package.json b/www/package.json index af1434e0..ee03840a 100644 --- a/www/package.json +++ b/www/package.json @@ -14,7 +14,7 @@ "@fortawesome/fontawesome-svg-core": "^6.4.0", "@fortawesome/free-solid-svg-icons": "^6.4.0", "@fortawesome/react-fontawesome": "^0.2.0", - "@sentry/nextjs": "^7.61.0", + "@sentry/nextjs": "^7.64.0", "autoprefixer": "10.4.14", "axios": "^1.4.0", "fontawesome": "^5.6.3", diff --git a/www/pages/api/sentry-example-api.js b/www/pages/api/sentry-example-api.js new file mode 100644 index 00000000..ac07eec0 --- /dev/null +++ b/www/pages/api/sentry-example-api.js @@ -0,0 +1,5 @@ +// A faulty API route to test Sentry's error monitoring +export default function handler(_req, res) { + throw new Error("Sentry Example API Route Error"); + res.status(200).json({ name: "John Doe" }); +} diff --git a/www/pages/sentry-example-page.js b/www/pages/sentry-example-page.js new file mode 100644 index 00000000..bcace78b --- /dev/null +++ b/www/pages/sentry-example-page.js @@ -0,0 +1,87 @@ +import Head from "next/head"; +import * as Sentry from "@sentry/nextjs"; + +export default function Home() { + return ( +
+ + Sentry Onboarding + + + +
+

+ + + +

+ +

Get started by sending us a sample error:

+ + +

+ Next, look for the error on the{" "} + + Issues Page + + . +

+

+ For more information, see{" "} + + https://docs.sentry.io/platforms/javascript/guides/nextjs/ + +

+
+
+ ); +} diff --git a/www/sentry.client.config.js b/www/sentry.client.config.ts similarity index 100% rename from www/sentry.client.config.js rename to www/sentry.client.config.ts diff --git a/www/sentry.edge.config.js b/www/sentry.edge.config.ts similarity index 100% rename from www/sentry.edge.config.js rename to www/sentry.edge.config.ts diff --git a/www/sentry.server.config.js b/www/sentry.server.config.ts similarity index 100% rename from www/sentry.server.config.js rename to www/sentry.server.config.ts diff --git a/www/yarn.lock b/www/yarn.lock index 27d8c54e..d2ae9035 100644 --- a/www/yarn.lock +++ b/www/yarn.lock @@ -252,26 +252,26 @@ estree-walker "^2.0.2" picomatch "^2.3.1" -"@sentry-internal/tracing@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.61.0.tgz#5a0dd4a9a0b41f2e22904430f3fe0216f36ee086" - integrity sha512-zTr+MXEG4SxNxif42LIgm2RQn+JRXL2NuGhRaKSD2i4lXKFqHVGlVdoWqY5UfqnnJPokiTWIj9ejR8I5HV8Ogw== +"@sentry-internal/tracing@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.64.0.tgz#3e110473b8edf805b799cc91d6ee592830237bb4" + integrity sha512-1XE8W6ki7hHyBvX9hfirnGkKDBKNq3bDJyXS86E0bYVDl94nvbRM9BD9DHsCFetqYkVm1yDGEK+6aUVs4CztoQ== dependencies: - "@sentry/core" "7.61.0" - "@sentry/types" "7.61.0" - "@sentry/utils" "7.61.0" + "@sentry/core" "7.64.0" + "@sentry/types" "7.64.0" + "@sentry/utils" "7.64.0" tslib "^2.4.1 || ^1.9.3" -"@sentry/browser@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/browser/-/browser-7.61.0.tgz#04f4122e444d8b5ffefed97af3cde2bc1c71bb80" - integrity sha512-IGEkJZRP16Oe5CkXkmhU3QdV5RugW6Vds16yJFFYsgp87NprWtRZgqzldFDYkINStfBHVdctj/Rh/ZrLf8QlkQ== +"@sentry/browser@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/browser/-/browser-7.64.0.tgz#76db08a5d32ffe7c5aa907f258e6c845ce7f10d7" + integrity sha512-lB2IWUkZavEDclxfLBp554dY10ZNIEvlDZUWWathW+Ws2wRb6PNLtuPUNu12R7Q7z0xpkOLrM1kRNN0OdldgKA== dependencies: - "@sentry-internal/tracing" "7.61.0" - "@sentry/core" "7.61.0" - "@sentry/replay" "7.61.0" - "@sentry/types" "7.61.0" - "@sentry/utils" "7.61.0" + "@sentry-internal/tracing" "7.64.0" + "@sentry/core" "7.64.0" + "@sentry/replay" "7.64.0" + "@sentry/types" "7.64.0" + "@sentry/utils" "7.64.0" tslib "^2.4.1 || ^1.9.3" "@sentry/cli@^1.74.6": @@ -286,88 +286,88 @@ proxy-from-env "^1.1.0" which "^2.0.2" -"@sentry/core@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.61.0.tgz#0de4f73055bd156c5c0cbac50bb814b272567188" - integrity sha512-zl0ZKRjIoYJQWYTd3K/U6zZfS4GDY9yGd2EH4vuYO4kfYtEp/nJ8A+tfAeDo0c9FGxZ0Q+5t5F4/SfwbgyyQzg== +"@sentry/core@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.64.0.tgz#9d61cdc29ba299dedbdcbe01cfadf94bd0b7df48" + integrity sha512-IzmEyl5sNG7NyEFiyFHEHC+sizsZp9MEw1+RJRLX6U5RITvcsEgcajSkHQFafaBPzRrcxZMdm47Cwhl212LXcw== dependencies: - "@sentry/types" "7.61.0" - "@sentry/utils" "7.61.0" + "@sentry/types" "7.64.0" + "@sentry/utils" "7.64.0" tslib "^2.4.1 || ^1.9.3" -"@sentry/integrations@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/integrations/-/integrations-7.61.0.tgz#49c97a59ceb0438bd5ec070415d95f8c6c708d5f" - integrity sha512-NEQ+CatBfUM1TmA4FOOyHfsMvSIwSg4pA55Lxiq9quDykzkEtrXFzUfFpZbTunz4cegG8hucPOqbzKFrDPfGjQ== +"@sentry/integrations@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/integrations/-/integrations-7.64.0.tgz#a392ddeebeec0c08ae5ca1f544c80ab15977fe10" + integrity sha512-6gbSGiruOifAmLtXw//Za19GWiL5qugDMEFxSvc5WrBWb+A8UK+foPn3K495OcivLS68AmqAQCUGb+6nlVowwA== dependencies: - "@sentry/types" "7.61.0" - "@sentry/utils" "7.61.0" + "@sentry/types" "7.64.0" + "@sentry/utils" "7.64.0" localforage "^1.8.1" tslib "^2.4.1 || ^1.9.3" -"@sentry/nextjs@^7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/nextjs/-/nextjs-7.61.0.tgz#5a30faa6fb04d9148853edbb5c148dd522126097" - integrity sha512-zSEcAITqVmJpR4hhah1jUyCzm/hjlq9vjmO6BmTnQjr84OgOdeKJGWtRdktXId+9zzHdCOehs/JPtmO7y+yG6Q== +"@sentry/nextjs@^7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/nextjs/-/nextjs-7.64.0.tgz#5c0bd7ccc6637e0b925dec25ca247dcb8476663c" + integrity sha512-hKlIQpFugdRlWj0wcEG9I8JyVm/osdsE72zwMBGnmCw/jf7U63vjOjfxMe/gRuvllCf/AvoGHEkR5jPufcO+bw== dependencies: "@rollup/plugin-commonjs" "24.0.0" - "@sentry/core" "7.61.0" - "@sentry/integrations" "7.61.0" - "@sentry/node" "7.61.0" - "@sentry/react" "7.61.0" - "@sentry/types" "7.61.0" - "@sentry/utils" "7.61.0" + "@sentry/core" "7.64.0" + "@sentry/integrations" "7.64.0" + "@sentry/node" "7.64.0" + "@sentry/react" "7.64.0" + "@sentry/types" "7.64.0" + "@sentry/utils" "7.64.0" "@sentry/webpack-plugin" "1.20.0" chalk "3.0.0" rollup "2.78.0" stacktrace-parser "^0.1.10" tslib "^2.4.1 || ^1.9.3" -"@sentry/node@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/node/-/node-7.61.0.tgz#1309330f2ad136af532ad2a03b2a312e885705de" - integrity sha512-oTCqD/h92uvbRCrtCdiAqN6Mfe3vF7ywVHZ8Nq3hHmJp6XadUT+fCBwNQ7rjMyqJAOYAnx/vp6iN9n8C5qcYZQ== +"@sentry/node@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/node/-/node-7.64.0.tgz#c6f7a67c1442324298f0525e7191bc18572ee1ce" + integrity sha512-wRi0uTnp1WSa83X2yLD49tV9QPzGh5e42IKdIDBiQ7lV9JhLILlyb34BZY1pq6p4dp35yDasDrP3C7ubn7wo6A== dependencies: - "@sentry-internal/tracing" "7.61.0" - "@sentry/core" "7.61.0" - "@sentry/types" "7.61.0" - "@sentry/utils" "7.61.0" + "@sentry-internal/tracing" "7.64.0" + "@sentry/core" "7.64.0" + "@sentry/types" "7.64.0" + "@sentry/utils" "7.64.0" cookie "^0.4.1" https-proxy-agent "^5.0.0" lru_map "^0.3.3" tslib "^2.4.1 || ^1.9.3" -"@sentry/react@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/react/-/react-7.61.0.tgz#21dc8eb5168fdb45f994e62738313b50c710d6a4" - integrity sha512-17ZPDdzx3hzJSHsVFAiw4hUT701LUVIcm568q38sPlSUmnOmNmPeHx/xcQkuxMoVsw/xgf/82B/BKKnIP5/diA== +"@sentry/react@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/react/-/react-7.64.0.tgz#edee24ac232990204e0fb43dd83994642d4b0f54" + integrity sha512-wOyJUQi7OoT1q+F/fVVv1fzbyO4OYbTu6m1DliLOGQPGEHPBsgPc722smPIExd1/rAMK/FxOuNN5oNhubH8nhg== dependencies: - "@sentry/browser" "7.61.0" - "@sentry/types" "7.61.0" - "@sentry/utils" "7.61.0" + "@sentry/browser" "7.64.0" + "@sentry/types" "7.64.0" + "@sentry/utils" "7.64.0" hoist-non-react-statics "^3.3.2" tslib "^2.4.1 || ^1.9.3" -"@sentry/replay@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/replay/-/replay-7.61.0.tgz#f816d6a2fc7511877efee2e328681d659433d147" - integrity sha512-1ugk0yZssOPkSg6uTVcysjxlBydycXiOgV0PCU7DsXCFOV1ua5YpyPZFReTz9iFTtwD0LwGFM1LW9wJeQ67Fzg== +"@sentry/replay@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/replay/-/replay-7.64.0.tgz#bdf09b0c4712f9dc6b24b3ebefa55a4ac76708e6" + integrity sha512-alaMCZDZhaAVmEyiUnszZnvfdbiZx5MmtMTGrlDd7tYq3K5OA9prdLqqlmfIJYBfYtXF3lD0iZFphOZQD+4CIw== dependencies: - "@sentry/core" "7.61.0" - "@sentry/types" "7.61.0" - "@sentry/utils" "7.61.0" + "@sentry/core" "7.64.0" + "@sentry/types" "7.64.0" + "@sentry/utils" "7.64.0" -"@sentry/types@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.61.0.tgz#4243b5ef4658f6b0673bc4372c90e6ec920f78d8" - integrity sha512-/GLlIBNR35NKPE/SfWi9W10dK9hE8qTShzsuPVn5wAJxpT3Lb4+dkwmKCTLUYxdkmvRDEudkfOxgalsfQGTAWA== +"@sentry/types@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.64.0.tgz#21fc545ea05c3c8c4c3e518583eca1a8c5429506" + integrity sha512-LqjQprWXjUFRmzIlUjyA+KL+38elgIYmAeoDrdyNVh8MK5IC1W2Lh1Q87b4yOiZeMiIhIVNBd7Ecoh2rodGrGA== -"@sentry/utils@7.61.0": - version "7.61.0" - resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.61.0.tgz#16944afb2b851af045fb528c0c35b7dea3e1cd3b" - integrity sha512-jfj14d0XBFiCU0G6dZZ12SizATiF5Mt4stBGzkM5iS9nXFj8rh1oTT7/p+aZoYzP2JTF+sDzkNjWxyKZkcTo0Q== +"@sentry/utils@7.64.0": + version "7.64.0" + resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.64.0.tgz#6fe3ce9a56d3433ed32119f914907361a54cc184" + integrity sha512-HRlM1INzK66Gt+F4vCItiwGKAng4gqzCR4C5marsL3qv6SrKH98dQnCGYgXluSWaaa56h97FRQu7TxCk6jkSvQ== dependencies: - "@sentry/types" "7.61.0" + "@sentry/types" "7.64.0" tslib "^2.4.1 || ^1.9.3" "@sentry/webpack-plugin@1.20.0": From 5e2afa599e8d80cc6065b9dc85400056618bceb0 Mon Sep 17 00:00:00 2001 From: Koper Date: Wed, 16 Aug 2023 17:59:01 +0700 Subject: [PATCH 04/31] Deleted sentry example pages --- www/pages/api/sentry-example-api.js | 5 -- www/pages/sentry-example-page.js | 87 ----------------------------- 2 files changed, 92 deletions(-) delete mode 100644 www/pages/api/sentry-example-api.js delete mode 100644 www/pages/sentry-example-page.js diff --git a/www/pages/api/sentry-example-api.js b/www/pages/api/sentry-example-api.js deleted file mode 100644 index ac07eec0..00000000 --- a/www/pages/api/sentry-example-api.js +++ /dev/null @@ -1,5 +0,0 @@ -// A faulty API route to test Sentry's error monitoring -export default function handler(_req, res) { - throw new Error("Sentry Example API Route Error"); - res.status(200).json({ name: "John Doe" }); -} diff --git a/www/pages/sentry-example-page.js b/www/pages/sentry-example-page.js deleted file mode 100644 index bcace78b..00000000 --- a/www/pages/sentry-example-page.js +++ /dev/null @@ -1,87 +0,0 @@ -import Head from "next/head"; -import * as Sentry from "@sentry/nextjs"; - -export default function Home() { - return ( -
- - Sentry Onboarding - - - -
-

- - - -

- -

Get started by sending us a sample error:

- - -

- Next, look for the error on the{" "} - - Issues Page - - . -

-

- For more information, see{" "} - - https://docs.sentry.io/platforms/javascript/guides/nextjs/ - -

-
-
- ); -} From e12f9afe7b4283a7f11e95d39845d406ef14eb43 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Wed, 16 Aug 2023 17:24:05 +0200 Subject: [PATCH 05/31] server: implement user authentication (none by default) --- server/env.example | 14 +++++ server/poetry.lock | 66 ++++++++++++++++++++++- server/pyproject.toml | 1 + server/reflector/app.py | 1 + server/reflector/auth/__init__.py | 13 +++++ server/reflector/auth/auth_fief.py | 25 +++++++++ server/reflector/auth/auth_none.py | 35 ++++++++++++ server/reflector/settings.py | 8 +++ server/reflector/views/transcripts.py | 68 ++++++++++++++++++------ server/test.db | Bin 20480 -> 0 bytes server/tests/conftest.py | 16 ++++++ server/tests/test_transcripts.py | 35 +++++++++++- server/tests/test_transcripts_rtc_ws.py | 2 +- 13 files changed, 263 insertions(+), 21 deletions(-) create mode 100644 server/reflector/auth/__init__.py create mode 100644 server/reflector/auth/auth_fief.py create mode 100644 server/reflector/auth/auth_none.py delete mode 100644 server/test.db create mode 100644 server/tests/conftest.py diff --git a/server/env.example b/server/env.example index 11e0927b..a8fd4128 100644 --- a/server/env.example +++ b/server/env.example @@ -11,6 +11,20 @@ #DATABASE_URL=postgresql://reflector:reflector@localhost:5432/reflector +## ======================================================= +## User authentication +## ======================================================= + +## No authentication +#AUTH_BACKEND=none + +## Using fief (fief.dev) +#AUTH_BACKEND=fief +#AUTH_FIEF_URL=https://your-fief-instance.... +#AUTH_FIEF_CLIENT_ID=xxx +#AUTH_FIEF_CLIENT_SECRET=xxx + + ## ======================================================= ## Transcription backend ## diff --git a/server/poetry.lock b/server/poetry.lock index 9ad03bcf..b3b122f4 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -930,6 +930,23 @@ mysql = ["aiomysql"] postgresql = ["asyncpg"] sqlite = ["aiosqlite"] +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + [[package]] name = "dnspython" version = "2.4.1" @@ -1022,6 +1039,28 @@ tokenizers = "==0.13.*" conversion = ["transformers[torch] (>=4.23)"] dev = ["black (==23.*)", "flake8 (==6.*)", "isort (==5.*)", "pytest (==7.*)"] +[[package]] +name = "fief-client" +version = "0.17.0" +description = "Fief Client for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fief_client-0.17.0-py3-none-any.whl", hash = "sha256:ecc8674ecaf58fc7d2926f5a0f49fabd3a1a03e278f030977a97ecb716b8884d"}, + {file = "fief_client-0.17.0.tar.gz", hash = "sha256:f1f9a10c760c29811a8cce2c1d58938090901772826dda973b67dde1bce3bafd"}, +] + +[package.dependencies] +fastapi = {version = "*", optional = true, markers = "extra == \"fastapi\""} +httpx = ">=0.21.3,<0.25.0" +jwcrypto = ">=1.4,<2.0.0" +makefun = {version = ">=1.14.0,<2.0.0", optional = true, markers = "extra == \"fastapi\""} + +[package.extras] +cli = ["halo"] +fastapi = ["fastapi", "makefun (>=1.14.0,<2.0.0)"] +flask = ["flask"] + [[package]] name = "filelock" version = "3.12.2" @@ -1530,6 +1569,20 @@ files = [ {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] +[[package]] +name = "jwcrypto" +version = "1.5.0" +description = "Implementation of JOSE Web standards" +optional = false +python-versions = ">= 3.6" +files = [ + {file = "jwcrypto-1.5.0.tar.gz", hash = "sha256:2c1dc51cf8e38ddf324795dfe9426dee9dd46caf47f535ccbc18781fba810b8d"}, +] + +[package.dependencies] +cryptography = ">=3.4" +deprecated = "*" + [[package]] name = "levenshtein" version = "0.21.1" @@ -1662,6 +1715,17 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] dev = ["Sphinx (==5.3.0)", "colorama (==0.4.5)", "colorama (==0.4.6)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v0.990)", "pre-commit (==3.2.1)", "pytest (==6.1.2)", "pytest (==7.2.1)", "pytest-cov (==2.12.1)", "pytest-cov (==4.0.0)", "pytest-mypy-plugins (==1.10.1)", "pytest-mypy-plugins (==1.9.3)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.2.0)", "tox (==3.27.1)", "tox (==4.4.6)"] +[[package]] +name = "makefun" +version = "1.15.1" +description = "Small library to dynamically create python functions." +optional = false +python-versions = "*" +files = [ + {file = "makefun-1.15.1-py2.py3-none-any.whl", hash = "sha256:a63cfc7b47a539c76d97bd4fdb833c7d0461e759fd1225f580cb4be6200294d4"}, + {file = "makefun-1.15.1.tar.gz", hash = "sha256:40b0f118b6ded0d8d78c78f1eb679b8b6b2462e3c1b3e05fb1b2da8cd46b48a5"}, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -3234,4 +3298,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "ea523f9b74581a7867097a6249d416d8836f4daaf33fde65ea343e4d3502c71c" +content-hash = "d84edfea8ac7a849340af8eb5db47df9c13a7cc1c640062ebedb2a808be0de4e" diff --git a/server/pyproject.toml b/server/pyproject.toml index e3e75843..895be79d 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -25,6 +25,7 @@ httpx = "^0.24.1" fastapi-pagination = "^0.12.6" databases = {extras = ["aiosqlite", "asyncpg"], version = "^0.7.0"} sqlalchemy = "<1.5" +fief-client = {extras = ["fastapi"], version = "^0.17.0"} [tool.poetry.group.dev.dependencies] diff --git a/server/reflector/app.py b/server/reflector/app.py index 8383bf32..fa148240 100644 --- a/server/reflector/app.py +++ b/server/reflector/app.py @@ -3,6 +3,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi_pagination import add_pagination from fastapi.routing import APIRoute import reflector.db # noqa +import reflector.auth # noqa from reflector.views.rtc_offer import router as rtc_offer_router from reflector.views.transcripts import router as transcripts_router from reflector.events import subscribers_startup, subscribers_shutdown diff --git a/server/reflector/auth/__init__.py b/server/reflector/auth/__init__.py new file mode 100644 index 00000000..65e75d9b --- /dev/null +++ b/server/reflector/auth/__init__.py @@ -0,0 +1,13 @@ +from reflector.settings import settings +from reflector.logger import logger +import importlib + +logger.info(f"User authentication using {settings.AUTH_BACKEND}") +module_name = f"reflector.auth.auth_{settings.AUTH_BACKEND}" +auth_module = importlib.import_module(module_name) + +UserInfo = auth_module.UserInfo +AccessTokenInfo = auth_module.AccessTokenInfo +authenticated = auth_module.authenticated +current_user = auth_module.current_user +current_user_optional = auth_module.current_user_optional diff --git a/server/reflector/auth/auth_fief.py b/server/reflector/auth/auth_fief.py new file mode 100644 index 00000000..0b363fc0 --- /dev/null +++ b/server/reflector/auth/auth_fief.py @@ -0,0 +1,25 @@ +from fastapi.security import OAuth2AuthorizationCodeBearer +from fief_client import FiefAccessTokenInfo, FiefAsync, FiefUserInfo +from fief_client.integrations.fastapi import FiefAuth +from reflector.settings import settings + +fief = FiefAsync( + settings.AUTH_FIEF_URL, + settings.AUTH_FIEF_CLIENT_ID, + settings.AUTH_FIEF_CLIENT_SECRET, +) + +scheme = OAuth2AuthorizationCodeBearer( + f"{settings.AUTH_FIEF_URL}/authorize", + f"{settings.AUTH_FIEF_URL}/api/token", + scopes={"openid": "openid", "offline_access": "offline_access"}, + auto_error=False, +) + +auth = FiefAuth(fief, scheme) + +UserInfo = FiefUserInfo +AccessTokenInfo = FiefAccessTokenInfo +authenticated = auth.authenticated() +current_user = auth.current_user() +current_user_optional = auth.current_user(optional=True) diff --git a/server/reflector/auth/auth_none.py b/server/reflector/auth/auth_none.py new file mode 100644 index 00000000..3959c739 --- /dev/null +++ b/server/reflector/auth/auth_none.py @@ -0,0 +1,35 @@ +from pydantic import BaseModel +from typing import Annotated +from fastapi import Depends +from fastapi.security import OAuth2PasswordBearer + +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") + + +class UserInfo(BaseModel): + sub: str + + +class AccessTokenInfo(BaseModel): + pass + + +def authenticated(token: Annotated[str, Depends(oauth2_scheme)]): + def _authenticated(): + return None + + return _authenticated + + +def current_user(token: Annotated[str, Depends(oauth2_scheme)]): + def _current_user(): + return None + + return _current_user + + +def current_user_optional(token: Annotated[str, Depends(oauth2_scheme)]): + def _current_user_optional(): + return None + + return _current_user_optional diff --git a/server/reflector/settings.py b/server/reflector/settings.py index 0787b466..2add7448 100644 --- a/server/reflector/settings.py +++ b/server/reflector/settings.py @@ -79,5 +79,13 @@ class Settings(BaseSettings): # Sentry SENTRY_DSN: str | None = None + # User authentication (none, fief) + AUTH_BACKEND: str = "none" + + # User authentication using fief + AUTH_FIEF_URL: str | None = None + AUTH_FIEF_CLIENT_ID: str | None = None + AUTH_FIEF_CLIENT_SECRET: str | None = None + settings = Settings() diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 6f952938..778c47d7 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -4,6 +4,7 @@ from fastapi import ( Request, WebSocket, WebSocketDisconnect, + Depends, ) from fastapi.responses import FileResponse from starlette.concurrency import run_in_threadpool @@ -14,8 +15,9 @@ from fastapi_pagination import Page, paginate from reflector.logger import logger from reflector.db import database, transcripts from reflector.settings import settings +import reflector.auth as auth from .rtc_offer import rtc_offer_base, RtcOffer, PipelineEvent -from typing import Optional +from typing import Annotated, Optional from pathlib import Path from tempfile import NamedTemporaryFile import av @@ -60,6 +62,7 @@ class TranscriptEvent(BaseModel): class Transcript(BaseModel): id: str = Field(default_factory=generate_uuid4) + user_id: str | None = None name: str = Field(default_factory=generate_transcript_name) status: str = "idle" locked: bool = False @@ -127,20 +130,28 @@ class Transcript(BaseModel): class TranscriptController: - async def get_all(self) -> list[Transcript]: + async def get_all(self, user_id: str | None = None) -> list[Transcript]: query = transcripts.select() + if user_id is not None: + query = query.where(transcripts.c.user_id == user_id) + print(query) results = await database.fetch_all(query) + print(results) return results - async def get_by_id(self, transcript_id: str) -> Transcript | None: + async def get_by_id( + self, transcript_id: str, user_id: str | None = None + ) -> Transcript | None: query = transcripts.select().where(transcripts.c.id == transcript_id) result = await database.fetch_one(query) if not result: return None + if user_id is not None and result["user_id"] != user_id: + return None return Transcript(**result) - async def add(self, name: str): - transcript = Transcript(name=name) + async def add(self, name: str, user_id: str | None = None): + transcript = Transcript(name=name, user_id=user_id) query = transcripts.insert().values(**transcript.model_dump()) await database.execute(query) return transcript @@ -155,10 +166,14 @@ class TranscriptController: for key, value in values.items(): setattr(transcript, key, value) - async def remove_by_id(self, transcript_id: str) -> None: + async def remove_by_id( + self, transcript_id: str, user_id: str | None = None + ) -> None: transcript = await self.get_by_id(transcript_id) if not transcript: return + if user_id is not None and transcript.user_id != user_id: + return transcript.unlink() query = transcripts.delete().where(transcripts.c.id == transcript_id) await database.execute(query) @@ -199,13 +214,19 @@ class DeletionStatus(BaseModel): @router.get("/transcripts", response_model=Page[GetTranscript]) -async def transcripts_list(): - return paginate(await transcripts_controller.get_all()) +async def transcripts_list( + user: auth.UserInfo = Depends(auth.current_user), +): + return paginate(await transcripts_controller.get_all(user_id=user["sub"])) @router.post("/transcripts", response_model=GetTranscript) -async def transcripts_create(info: CreateTranscript): - return await transcripts_controller.add(info.name) +async def transcripts_create( + info: CreateTranscript, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +): + user_id = user["sub"] if user else None + return await transcripts_controller.add(info.name, user_id=user_id) # ============================================================== @@ -214,16 +235,25 @@ async def transcripts_create(info: CreateTranscript): @router.get("/transcripts/{transcript_id}", response_model=GetTranscript) -async def transcript_get(transcript_id: str): - transcript = await transcripts_controller.get_by_id(transcript_id) +async def transcript_get( + transcript_id: str, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +): + user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") return transcript @router.patch("/transcripts/{transcript_id}", response_model=GetTranscript) -async def transcript_update(transcript_id: str, info: UpdateTranscript): - transcript = await transcripts_controller.get_by_id(transcript_id) +async def transcript_update( + transcript_id: str, + info: UpdateTranscript, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +): + user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") values = {} @@ -236,11 +266,15 @@ async def transcript_update(transcript_id: str, info: UpdateTranscript): @router.delete("/transcripts/{transcript_id}", response_model=DeletionStatus) -async def transcript_delete(transcript_id: str): - transcript = await transcripts_controller.get_by_id(transcript_id) +async def transcript_delete( + transcript_id: str, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +): + user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") - await transcripts_controller.remove_by_id(transcript.id) + await transcripts_controller.remove_by_id(transcript.id, user_id=user_id) return DeletionStatus(status="ok") diff --git a/server/test.db b/server/test.db deleted file mode 100644 index 974a6a14e9005e55574c4cea2c0b7f5b04ae13d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeI2OKerWq?jB#3YHy&qm6oM5#9Cjr+J)e2Gcy)H83(RwCT z5Qs&qvhEHEv8<5T01}%bv8hrEfNb>Jt%FTZiK6pQb~2h-!H zk0$Z(?y;{)^=lWOj&M?EI9Y_6T1|D; zOAn{gL%wAKnLs9x31kA9Kqin0WCEE$CXfka0-3B#?gp-&p!VYw(l7ZvS8X5Bt}9pY(pzyVCtz_k*tK{JHaf z=X32pw7=W_%-ZkP-dVe_`kU4Jt7lezwGvj^Cx3o&y!`LwA207L{R1TOEfdHDGJ#Kn z!2U*OeE#g}>Q?LY=@xOE7gj0cyt9Z)rjgZ{BcoWbP6$Pq*^86uaCh(OLAVzwrVL?? zh!_&SN#!ObBdn;_{PvyOcitR!#^;Wtu~J@$AW>nhMw}^&v|Jkm;mN~wIIrmO@Z!hk5_HXt6HLRJ!tY>YYzLM0EnQYQ12sUW5Dtyeqa^&_7( zmW*QUkQJmr+(?UzGZay(gTyg<;$%{u?=k0`(d6;?d}n<2NS9mVuwf2GY6r93Nol`_1!DwS`&REwU)P6B<87_lC43QQ~oI>r;Pm3M`p zbE8-4Dr1CTW%ORGMbBvBA_k8G=Mi*+Lt1--xG+Rp5lj){oaj}uiff`v#OpN>M1V^i zJH(^`5QPC*R|KRKp;k*ve3fjo3xsSEGtxpDDof=TYRZ&Gp{yr}D+z8!Cy^mVL`*2g zA>bm!Vm{(0%8;*RBHub&L#zv{nKuQpOaTfKpq^t0sEE!LMU=c?rShz#sB{^o+?0r) zuOYUA1;P|~TxfW3fY@OikidjTLlsnEnQk)*q>?3s@-mBWovFLaioi5<3IfT6fOV=n zERluwCYhztbCN*JahHuS<3d#KK38)WdmRX2khvTKU?-vQw6};wE-Yu>hV-AlGwT^| z3OXW`LGF|kPS<3uaF{US5Wy7OHBf#=Q;9&@2ZABFjhQV=13_0jgNH`I}YbiW_GVJF2Or_za&*M&wa)E7BFxJ3H4n9W{pr1iv2nZWDE(k8m zVfVk4Q8~;p)3TCn*JM2un}MM;LVQGR8 z5v3prbxOC{!7G)S%iZ6}(Cr z8WpK3?_`ZKbT6uuFeo?)lmRTXH6@^`8MahP=lQ%6DP^G>m?zzGxh5-ARKP-^U=6e% z4gJ_@NDR&jD9C6q^(EyQIZ?@Cf^pd`$4fQYXn5gh(8zlWGaphJS?>tKm|7U0)F@nb z%h|GIBnZ5F)zubjvV{W*$y*&fKnPwy)p@v#Hf&kGsZ#Abvg#=7)2{r%^?Wjf<s zq^~5yCQHwA$*{=|^W|jNWF`4h`PL0~j4vj`CR@YxWY}a0_(C#lveP@844W+Jo-Y*} ztliEe!zO#Q=c>1U`ey8OGHkNzI+YBYY_A5%u*p8EpA4I9mwKg-4VFdSWY}c)(@BO+ zwmI!&*knJmmJFM0TUL`{ljX@uGHkL7IhhQbY&Vum9~ Date: Wed, 16 Aug 2023 22:37:20 +0530 Subject: [PATCH 06/31] make schema optional for all LLMs --- server/gpu/modal/reflector_llm.py | 44 +++++++++++-------- server/reflector/llm/base.py | 2 +- server/reflector/llm/llm_banana.py | 12 +++-- server/reflector/llm/llm_modal.py | 21 +++++++-- server/reflector/llm/llm_oobagooda.py | 13 ++++-- .../processors/transcript_topic_detector.py | 18 ++++++-- server/reflector/server.py | 2 +- server/reflector/settings.py | 4 +- 8 files changed, 79 insertions(+), 37 deletions(-) diff --git a/server/gpu/modal/reflector_llm.py b/server/gpu/modal/reflector_llm.py index 315ff785..d83d5036 100644 --- a/server/gpu/modal/reflector_llm.py +++ b/server/gpu/modal/reflector_llm.py @@ -5,8 +5,9 @@ Reflector GPU backend - LLM """ import os -from modal import Image, method, Stub, asgi_app, Secret +from modal import asgi_app, Image, method, Secret, Stub +from pydantic.typing import Optional # LLM LLM_MODEL: str = "lmsys/vicuna-13b-v1.5" @@ -100,13 +101,6 @@ class LLM: self.model = model self.tokenizer = tokenizer self.gen_cfg = gen_cfg - self.json_schema = { - "type": "object", - "properties": { - "title": {"type": "string"}, - "summary": {"type": "string"}, - }, - } def __exit__(self, *args): print("Exit llm") @@ -117,19 +111,30 @@ class LLM: return {"status": "ok"} @method() - def generate(self, prompt: str): + def generate(self, prompt: str, schema: str = None): print(f"Generate {prompt=}") - import jsonformer - import json + if schema: + import ast + import jsonformer - jsonformer_llm = jsonformer.Jsonformer(model=self.model, - tokenizer=self.tokenizer, - json_schema=self.json_schema, - prompt=prompt, - max_string_token_length=self.gen_cfg.max_new_tokens) - response = jsonformer_llm() + jsonformer_llm = jsonformer.Jsonformer(model=self.model, + tokenizer=self.tokenizer, + json_schema=ast.literal_eval(schema), + prompt=prompt, + max_string_token_length=self.gen_cfg.max_new_tokens) + response = jsonformer_llm() + print(f"Generated {response=}") + return {"text": response} + + input_ids = self.tokenizer.encode(prompt, return_tensors="pt").to( + self.model.device + ) + output = self.model.generate(input_ids, generation_config=self.gen_cfg) + + # decode output + response = self.tokenizer.decode(output[0].cpu(), skip_special_tokens=True) print(f"Generated {response=}") - return {"text": json.dumps(response)} + return {"text": response} # ------------------------------------------------------------------- @@ -165,12 +170,13 @@ def web(): class LLMRequest(BaseModel): prompt: str + schema: Optional[str] = None @app.post("/llm", dependencies=[Depends(apikey_auth)]) async def llm( req: LLMRequest, ): - func = llmstub.generate.spawn(prompt=req.prompt) + func = llmstub.generate.spawn(prompt=req.prompt, schema=req.schema) result = func.get() return result diff --git a/server/reflector/llm/base.py b/server/reflector/llm/base.py index e528a3e6..5e86b553 100644 --- a/server/reflector/llm/base.py +++ b/server/reflector/llm/base.py @@ -20,7 +20,7 @@ class LLM: Return an instance depending on the settings. Settings used: - - `LLM_BACKEND`: key of the backend, defaults to `oobagooda` + - `LLM_BACKEND`: key of the backend, defaults to `oobabooga` - `LLM_URL`: url of the backend """ if name is None: diff --git a/server/reflector/llm/llm_banana.py b/server/reflector/llm/llm_banana.py index d6a0fa07..bdaf091f 100644 --- a/server/reflector/llm/llm_banana.py +++ b/server/reflector/llm/llm_banana.py @@ -1,7 +1,9 @@ +import json + +import httpx from reflector.llm.base import LLM from reflector.settings import settings from reflector.utils.retry import retry -import httpx class BananaLLM(LLM): @@ -14,17 +16,21 @@ class BananaLLM(LLM): } async def _generate(self, prompt: str, **kwargs): + json_payload = {"prompt": prompt} + if "schema" in kwargs: + json_payload["schema"] = json.dumps(kwargs["schema"]) async with httpx.AsyncClient() as client: response = await retry(client.post)( settings.LLM_URL, headers=self.headers, - json={"prompt": prompt}, + json=json_payload, timeout=self.timeout, retry_timeout=300, # as per their sdk ) response.raise_for_status() text = response.json()["text"] - text = text[len(prompt) :] # remove prompt + if "schema" not in json_payload: + text = text[len(prompt) :] return text diff --git a/server/reflector/llm/llm_modal.py b/server/reflector/llm/llm_modal.py index 7f23aa0d..692dd095 100644 --- a/server/reflector/llm/llm_modal.py +++ b/server/reflector/llm/llm_modal.py @@ -1,7 +1,9 @@ +import json + +import httpx from reflector.llm.base import LLM from reflector.settings import settings from reflector.utils.retry import retry -import httpx class ModalLLM(LLM): @@ -24,17 +26,21 @@ class ModalLLM(LLM): response.raise_for_status() async def _generate(self, prompt: str, **kwargs): + json_payload = {"prompt": prompt} + if "schema" in kwargs: + json_payload["schema"] = json.dumps(kwargs["schema"]) async with httpx.AsyncClient() as client: response = await retry(client.post)( self.llm_url, headers=self.headers, - json={"prompt": prompt}, + json=json_payload, timeout=self.timeout, retry_timeout=60 * 5, ) response.raise_for_status() text = response.json()["text"] - text = text[len(prompt) :] # remove prompt + if "schema" not in json_payload: + text = text[len(prompt) :] return text @@ -48,6 +54,15 @@ if __name__ == "__main__": result = await llm.generate("Hello, my name is", logger=logger) print(result) + kwargs = { + "schema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + } + } + result = await llm.generate("Hello, my name is", kwargs=kwargs, logger=logger) + print(result) + import asyncio asyncio.run(main()) diff --git a/server/reflector/llm/llm_oobagooda.py b/server/reflector/llm/llm_oobagooda.py index be7d8133..85306135 100644 --- a/server/reflector/llm/llm_oobagooda.py +++ b/server/reflector/llm/llm_oobagooda.py @@ -1,18 +1,23 @@ +import json + +import httpx from reflector.llm.base import LLM from reflector.settings import settings -import httpx -class OobagoodaLLM(LLM): +class OobaboogaLLM(LLM): async def _generate(self, prompt: str, **kwargs): + json_payload = {"prompt": prompt} + if "schema" in kwargs: + json_payload["schema"] = json.dumps(kwargs["schema"]) async with httpx.AsyncClient() as client: response = await client.post( settings.LLM_URL, headers={"Content-Type": "application/json"}, - json={"prompt": prompt}, + json=json_payload, ) response.raise_for_status() return response.json() -LLM.register("oobagooda", OobagoodaLLM) +LLM.register("oobabooga", OobaboogaLLM) diff --git a/server/reflector/processors/transcript_topic_detector.py b/server/reflector/processors/transcript_topic_detector.py index 6e926771..430e3992 100644 --- a/server/reflector/processors/transcript_topic_detector.py +++ b/server/reflector/processors/transcript_topic_detector.py @@ -1,7 +1,7 @@ -from reflector.processors.base import Processor -from reflector.processors.types import Transcript, TitleSummary -from reflector.utils.retry import retry from reflector.llm import LLM +from reflector.processors.base import Processor +from reflector.processors.types import TitleSummary, Transcript +from reflector.utils.retry import retry class TranscriptTopicDetectorProcessor(Processor): @@ -31,6 +31,14 @@ class TranscriptTopicDetectorProcessor(Processor): self.transcript = None self.min_transcript_length = min_transcript_length self.llm = LLM.get_instance() + self.topic_detector_schema = { + "type": "object", + "properties": { + "title": {"type": "string"}, + "summary": {"type": "string"}, + }, + } + self.kwargs = {"schema": self.topic_detector_schema} async def _warmup(self): await self.llm.warmup(logger=self.logger) @@ -53,7 +61,9 @@ class TranscriptTopicDetectorProcessor(Processor): text = self.transcript.text self.logger.info(f"Topic detector got {len(text)} length transcript") prompt = self.PROMPT.format(input_text=text) - result = await retry(self.llm.generate)(prompt=prompt, logger=self.logger) + result = await retry(self.llm.generate)( + prompt=prompt, kwargs=self.kwargs, logger=self.logger + ) summary = TitleSummary( title=result["title"], summary=result["summary"], diff --git a/server/reflector/server.py b/server/reflector/server.py index 8e28b583..3b09efe4 100644 --- a/server/reflector/server.py +++ b/server/reflector/server.py @@ -41,7 +41,7 @@ model = WhisperModel("tiny", device="cpu", compute_type="float32", num_workers=1 # LLM LLM_URL = settings.LLM_URL if not LLM_URL: - assert settings.LLM_BACKEND == "oobagooda" + assert settings.LLM_BACKEND == "oobabooga" LLM_URL = f"http://{settings.LLM_HOST}:{settings.LLM_PORT}/api/v1/generate" logger.info(f"Using LLM [{settings.LLM_BACKEND}]: {LLM_URL}") diff --git a/server/reflector/settings.py b/server/reflector/settings.py index e776875b..81f817da 100644 --- a/server/reflector/settings.py +++ b/server/reflector/settings.py @@ -52,8 +52,8 @@ class Settings(BaseSettings): TRANSCRIPT_STORAGE_AWS_SECRET_ACCESS_KEY: str | None = None # LLM - # available backend: openai, banana, modal, oobagooda - LLM_BACKEND: str = "oobagooda" + # available backend: openai, banana, modal, oobabooga + LLM_BACKEND: str = "oobabooga" # LLM common configuration LLM_URL: str | None = None From eb13a7bd64e83e5cadbd2151a34af3d733aaebd0 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Thu, 17 Aug 2023 09:23:14 +0530 Subject: [PATCH 07/31] make schema optional argument --- server/gpu/modal/reflector_llm.py | 2 +- server/reflector/llm/base.py | 18 ++++++++++------- server/reflector/llm/llm_banana.py | 9 +++++---- server/reflector/llm/llm_modal.py | 20 +++++++++---------- server/reflector/llm/llm_oobagooda.py | 7 ++++--- server/reflector/llm/llm_openai.py | 6 ++++-- .../processors/transcript_topic_detector.py | 3 +-- server/tests/test_processors_pipeline.py | 5 ++++- server/tests/test_transcripts_rtc_ws.py | 15 +++++++------- 9 files changed, 48 insertions(+), 37 deletions(-) diff --git a/server/gpu/modal/reflector_llm.py b/server/gpu/modal/reflector_llm.py index d83d5036..2f96e330 100644 --- a/server/gpu/modal/reflector_llm.py +++ b/server/gpu/modal/reflector_llm.py @@ -5,9 +5,9 @@ Reflector GPU backend - LLM """ import os +from typing import Optional from modal import asgi_app, Image, method, Secret, Stub -from pydantic.typing import Optional # LLM LLM_MODEL: str = "lmsys/vicuna-13b-v1.5" diff --git a/server/reflector/llm/base.py b/server/reflector/llm/base.py index 5e86b553..fddf185d 100644 --- a/server/reflector/llm/base.py +++ b/server/reflector/llm/base.py @@ -1,10 +1,12 @@ -from reflector.settings import settings -from reflector.utils.retry import retry -from reflector.logger import logger as reflector_logger -from time import monotonic import importlib import json import re +from time import monotonic +from typing import Union + +from reflector.logger import logger as reflector_logger +from reflector.settings import settings +from reflector.utils.retry import retry class LLM: @@ -44,10 +46,12 @@ class LLM: async def _warmup(self, logger: reflector_logger): pass - async def generate(self, prompt: str, logger: reflector_logger, **kwargs) -> dict: + async def generate( + self, prompt: str, logger: reflector_logger, schema: str = None, **kwargs + ) -> dict: logger.info("LLM generate", prompt=repr(prompt)) try: - result = await retry(self._generate)(prompt=prompt, **kwargs) + result = await retry(self._generate)(prompt=prompt, schema=schema, **kwargs) except Exception: logger.exception("Failed to call llm after retrying") raise @@ -59,7 +63,7 @@ class LLM: return result - async def _generate(self, prompt: str, **kwargs) -> str: + async def _generate(self, prompt: str, schema: Union[str | None], **kwargs) -> str: raise NotImplementedError def _parse_json(self, result: str) -> dict: diff --git a/server/reflector/llm/llm_banana.py b/server/reflector/llm/llm_banana.py index bdaf091f..473769cc 100644 --- a/server/reflector/llm/llm_banana.py +++ b/server/reflector/llm/llm_banana.py @@ -1,4 +1,5 @@ import json +from typing import Union import httpx from reflector.llm.base import LLM @@ -15,10 +16,10 @@ class BananaLLM(LLM): "X-Banana-Model-Key": settings.LLM_BANANA_MODEL_KEY, } - async def _generate(self, prompt: str, **kwargs): + async def _generate(self, prompt: str, schema: Union[str | None], **kwargs): json_payload = {"prompt": prompt} - if "schema" in kwargs: - json_payload["schema"] = json.dumps(kwargs["schema"]) + if schema: + json_payload["schema"] = json.dumps(schema) async with httpx.AsyncClient() as client: response = await retry(client.post)( settings.LLM_URL, @@ -29,7 +30,7 @@ class BananaLLM(LLM): ) response.raise_for_status() text = response.json()["text"] - if "schema" not in json_payload: + if not schema: text = text[len(prompt) :] return text diff --git a/server/reflector/llm/llm_modal.py b/server/reflector/llm/llm_modal.py index 692dd095..c1fb856b 100644 --- a/server/reflector/llm/llm_modal.py +++ b/server/reflector/llm/llm_modal.py @@ -1,4 +1,5 @@ import json +from typing import Union import httpx from reflector.llm.base import LLM @@ -25,10 +26,10 @@ class ModalLLM(LLM): ) response.raise_for_status() - async def _generate(self, prompt: str, **kwargs): + async def _generate(self, prompt: str, schema: Union[str | None], **kwargs): json_payload = {"prompt": prompt} - if "schema" in kwargs: - json_payload["schema"] = json.dumps(kwargs["schema"]) + if schema: + json_payload["schema"] = json.dumps(schema) async with httpx.AsyncClient() as client: response = await retry(client.post)( self.llm_url, @@ -39,7 +40,7 @@ class ModalLLM(LLM): ) response.raise_for_status() text = response.json()["text"] - if "schema" not in json_payload: + if not schema: text = text[len(prompt) :] return text @@ -54,13 +55,12 @@ if __name__ == "__main__": result = await llm.generate("Hello, my name is", logger=logger) print(result) - kwargs = { - "schema": { - "type": "object", - "properties": {"name": {"type": "string"}}, - } + schema = { + "type": "object", + "properties": {"name": {"type": "string"}}, } - result = await llm.generate("Hello, my name is", kwargs=kwargs, logger=logger) + + result = await llm.generate("Hello, my name is", schema=schema, logger=logger) print(result) import asyncio diff --git a/server/reflector/llm/llm_oobagooda.py b/server/reflector/llm/llm_oobagooda.py index 85306135..0ceb442d 100644 --- a/server/reflector/llm/llm_oobagooda.py +++ b/server/reflector/llm/llm_oobagooda.py @@ -1,4 +1,5 @@ import json +from typing import Union import httpx from reflector.llm.base import LLM @@ -6,10 +7,10 @@ from reflector.settings import settings class OobaboogaLLM(LLM): - async def _generate(self, prompt: str, **kwargs): + async def _generate(self, prompt: str, schema: Union[str | None], **kwargs): json_payload = {"prompt": prompt} - if "schema" in kwargs: - json_payload["schema"] = json.dumps(kwargs["schema"]) + if schema: + json_payload["schema"] = json.dumps(schema) async with httpx.AsyncClient() as client: response = await client.post( settings.LLM_URL, diff --git a/server/reflector/llm/llm_openai.py b/server/reflector/llm/llm_openai.py index dd438704..9a74e03c 100644 --- a/server/reflector/llm/llm_openai.py +++ b/server/reflector/llm/llm_openai.py @@ -1,7 +1,9 @@ +from typing import Union + +import httpx from reflector.llm.base import LLM from reflector.logger import logger from reflector.settings import settings -import httpx class OpenAILLM(LLM): @@ -15,7 +17,7 @@ class OpenAILLM(LLM): self.max_tokens = settings.LLM_MAX_TOKENS logger.info(f"LLM use openai backend at {self.openai_url}") - async def _generate(self, prompt: str, **kwargs) -> str: + async def _generate(self, prompt: str, schema: Union[str | None], **kwargs) -> str: headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.openai_key}", diff --git a/server/reflector/processors/transcript_topic_detector.py b/server/reflector/processors/transcript_topic_detector.py index 430e3992..9ae21a72 100644 --- a/server/reflector/processors/transcript_topic_detector.py +++ b/server/reflector/processors/transcript_topic_detector.py @@ -14,7 +14,6 @@ class TranscriptTopicDetectorProcessor(Processor): PROMPT = """ ### Human: - Generate information based on the given schema: For the title field, generate a short title for the given text. For the summary field, summarize the given text in a maximum of @@ -62,7 +61,7 @@ class TranscriptTopicDetectorProcessor(Processor): self.logger.info(f"Topic detector got {len(text)} length transcript") prompt = self.PROMPT.format(input_text=text) result = await retry(self.llm.generate)( - prompt=prompt, kwargs=self.kwargs, logger=self.logger + prompt=prompt, schema=self.topic_detector_schema, logger=self.logger ) summary = TitleSummary( title=result["title"], diff --git a/server/tests/test_processors_pipeline.py b/server/tests/test_processors_pipeline.py index 95c296de..56cac96e 100644 --- a/server/tests/test_processors_pipeline.py +++ b/server/tests/test_processors_pipeline.py @@ -9,13 +9,16 @@ async def test_basic_process(event_loop): from reflector.settings import settings from reflector.llm.base import LLM from pathlib import Path + from typing import Union # use an LLM test backend settings.LLM_BACKEND = "test" settings.TRANSCRIPT_BACKEND = "whisper" class LLMTest(LLM): - async def _generate(self, prompt: str, **kwargs) -> str: + async def _generate( + self, prompt: str, schema: Union[str | None], **kwargs + ) -> str: return { "title": "TITLE", "summary": "SUMMARY", diff --git a/server/tests/test_transcripts_rtc_ws.py b/server/tests/test_transcripts_rtc_ws.py index 70ee209b..23c7813f 100644 --- a/server/tests/test_transcripts_rtc_ws.py +++ b/server/tests/test_transcripts_rtc_ws.py @@ -3,17 +3,18 @@ # FIXME test websocket connection after RTC is finished still send the full events # FIXME try with locked session, RTC should not work -import pytest +import asyncio import json +import threading +from pathlib import Path +from typing import Union from unittest.mock import patch -from httpx import AsyncClient +import pytest +from httpx import AsyncClient +from httpx_ws import aconnect_ws from reflector.app import app from uvicorn import Config, Server -import threading -import asyncio -from pathlib import Path -from httpx_ws import aconnect_ws class ThreadedUvicorn: @@ -61,7 +62,7 @@ async def dummy_llm(): from reflector.llm.base import LLM class TestLLM(LLM): - async def _generate(self, prompt: str, **kwargs): + async def _generate(self, prompt: str, schema: Union[str | None], **kwargs): return json.dumps({"title": "LLM TITLE", "summary": "LLM SUMMARY"}) with patch("reflector.llm.base.LLM.get_instance") as mock_llm: From 2e48f89fdc0a8e9df82fe36864e3fe53935695e5 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Thu, 17 Aug 2023 09:33:59 +0530 Subject: [PATCH 08/31] add comments and log --- server/gpu/modal/reflector_llm.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/server/gpu/modal/reflector_llm.py b/server/gpu/modal/reflector_llm.py index 2f96e330..21306763 100644 --- a/server/gpu/modal/reflector_llm.py +++ b/server/gpu/modal/reflector_llm.py @@ -113,7 +113,9 @@ class LLM: @method() def generate(self, prompt: str, schema: str = None): print(f"Generate {prompt=}") + # If a schema is given, conform to schema if schema: + print(f"Schema {schema=}") import ast import jsonformer @@ -123,16 +125,17 @@ class LLM: prompt=prompt, max_string_token_length=self.gen_cfg.max_new_tokens) response = jsonformer_llm() - print(f"Generated {response=}") - return {"text": response} + else: + # If no schema, perform prompt only generation - input_ids = self.tokenizer.encode(prompt, return_tensors="pt").to( + # tokenize prompt + input_ids = self.tokenizer.encode(prompt, return_tensors="pt").to( self.model.device - ) - output = self.model.generate(input_ids, generation_config=self.gen_cfg) + ) + output = self.model.generate(input_ids, generation_config=self.gen_cfg) - # decode output - response = self.tokenizer.decode(output[0].cpu(), skip_special_tokens=True) + # decode output + response = self.tokenizer.decode(output[0].cpu(), skip_special_tokens=True) print(f"Generated {response=}") return {"text": response} From a24c3afe5bc94ed4f25b48254478006f11fbd7c8 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Thu, 17 Aug 2023 09:35:49 +0530 Subject: [PATCH 09/31] cleanup --- server/reflector/processors/transcript_topic_detector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/reflector/processors/transcript_topic_detector.py b/server/reflector/processors/transcript_topic_detector.py index 9ae21a72..3e984741 100644 --- a/server/reflector/processors/transcript_topic_detector.py +++ b/server/reflector/processors/transcript_topic_detector.py @@ -37,7 +37,6 @@ class TranscriptTopicDetectorProcessor(Processor): "summary": {"type": "string"}, }, } - self.kwargs = {"schema": self.topic_detector_schema} async def _warmup(self): await self.llm.warmup(logger=self.logger) From 235ee73f462e6e1a2def5d21168b5b41bfe4a029 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Thu, 17 Aug 2023 09:59:16 +0530 Subject: [PATCH 10/31] update prompt --- server/reflector/processors/transcript_topic_detector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/reflector/processors/transcript_topic_detector.py b/server/reflector/processors/transcript_topic_detector.py index 3e984741..3d8e3965 100644 --- a/server/reflector/processors/transcript_topic_detector.py +++ b/server/reflector/processors/transcript_topic_detector.py @@ -14,6 +14,8 @@ class TranscriptTopicDetectorProcessor(Processor): PROMPT = """ ### Human: + Create a JSON object as response.The JSON object must have 2 fields: + i) title and ii) summary. For the title field, generate a short title for the given text. For the summary field, summarize the given text in a maximum of From e297b8269d67c37ae4ebbf419138ecfd0e35d189 Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 17 Aug 2023 15:30:35 +0700 Subject: [PATCH 11/31] Removed unnecessary suffix "Type" --- www/app/transcripts/dashboard.tsx | 6 +++--- www/app/transcripts/useTranscript.tsx | 4 ++-- www/app/transcripts/useWebSockets.tsx | 20 +++++++++----------- www/app/transcripts/webSocketTypes.tsx | 8 ++++---- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/www/app/transcripts/dashboard.tsx b/www/app/transcripts/dashboard.tsx index b426c670..6fc2cfbe 100644 --- a/www/app/transcripts/dashboard.tsx +++ b/www/app/transcripts/dashboard.tsx @@ -6,15 +6,15 @@ import { } from "@fortawesome/free-solid-svg-icons"; import { formatTime } from "../lib/time"; import ScrollToBottom from "./scrollToBottom"; -import FinalSummary from "./finalSummary"; import DisconnectedIndicator from "./disconnectedIndicator"; import LiveTrancription from "./liveTranscription"; -import { TopicType, FinalSummaryType } from "./webSocketTypes"; +import FinalSummary from "./finalSummary"; +import { Topic, FinalSummary as FinalSummaryType } from "./webSocketTypes"; type DashboardProps = { transcriptionText: string; finalSummary: FinalSummaryType; - topics: TopicType[]; + topics: Topic[]; disconnected: boolean; }; diff --git a/www/app/transcripts/useTranscript.tsx b/www/app/transcripts/useTranscript.tsx index ddb77bed..11510020 100644 --- a/www/app/transcripts/useTranscript.tsx +++ b/www/app/transcripts/useTranscript.tsx @@ -3,14 +3,14 @@ import { DefaultApi, V1TranscriptsCreateRequest } from "../api/apis/DefaultApi"; import { Configuration } from "../api/runtime"; import { GetTranscript } from "../api"; -type UseTranscriptReturnType = { +type UseTranscript = { response: GetTranscript | null; loading: boolean; error: string | null; createTranscript: () => void; }; -const useTranscript = (): UseTranscriptReturnType => { +const useTranscript = (): UseTranscript => { const [response, setResponse] = useState(null); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); diff --git a/www/app/transcripts/useWebSockets.tsx b/www/app/transcripts/useWebSockets.tsx index 5e872ccb..80d4f560 100644 --- a/www/app/transcripts/useWebSockets.tsx +++ b/www/app/transcripts/useWebSockets.tsx @@ -1,22 +1,20 @@ import { useEffect, useState } from "react"; -import { TopicType, FinalSummaryType, StatusType } from "./webSocketTypes"; +import { Topic, FinalSummary, Status } from "./webSocketTypes"; -type UseWebSocketsReturnType = { +type UseWebSockets = { transcriptText: string; - topics: TopicType[]; - finalSummary: FinalSummaryType; - status: StatusType; + topics: Topic[]; + finalSummary: FinalSummary; + status: Status; }; -export const useWebSockets = ( - transcriptId: string | null, -): UseWebSocketsReturnType => { +export const useWebSockets = (transcriptId: string | null): UseWebSockets => { const [transcriptText, setTranscriptText] = useState(""); - const [topics, setTopics] = useState([]); - const [finalSummary, setFinalSummary] = useState({ + const [topics, setTopics] = useState([]); + const [finalSummary, setFinalSummary] = useState({ summary: "", }); - const [status, setStatus] = useState({ value: "disconnected" }); + const [status, setStatus] = useState({ value: "disconnected" }); useEffect(() => { if (!transcriptId) return; diff --git a/www/app/transcripts/webSocketTypes.tsx b/www/app/transcripts/webSocketTypes.tsx index 6ae6cac6..4b9358a5 100644 --- a/www/app/transcripts/webSocketTypes.tsx +++ b/www/app/transcripts/webSocketTypes.tsx @@ -1,4 +1,4 @@ -export type TopicType = { +export type Topic = { timestamp: number; title: string; transcript: string; @@ -6,14 +6,14 @@ export type TopicType = { id: string; }; -export type TranscriptType = { +export type Transcript = { text: string; }; -export type FinalSummaryType = { +export type FinalSummary = { summary: string; }; -export type StatusType = { +export type Status = { value: string; }; From a98a9853be364890c87ff8320b56f27b788b1e95 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Thu, 17 Aug 2023 14:42:45 +0530 Subject: [PATCH 12/31] PR review comments --- .pre-commit-config.yaml | 7 +++++++ server/env.example | 4 ++-- server/reflector/llm/base.py | 5 ++--- server/reflector/llm/llm_banana.py | 3 +-- server/reflector/llm/llm_modal.py | 3 +-- .../reflector/llm/{llm_oobagooda.py => llm_oobabooga.py} | 3 +-- server/reflector/llm/llm_openai.py | 4 +--- server/tests/test_processors_pipeline.py | 5 +---- server/tests/test_transcripts_rtc_ws.py | 3 +-- 9 files changed, 17 insertions(+), 20 deletions(-) rename server/reflector/llm/{llm_oobagooda.py => llm_oobabooga.py} (84%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7132e09c..2a73b7c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,3 +29,10 @@ repos: hooks: - id: black files: ^server/(reflector|tests)/ + + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) + files: ^server/(gpu|evaluate|reflector)/ diff --git a/server/env.example b/server/env.example index 11e0927b..2317a1bb 100644 --- a/server/env.example +++ b/server/env.example @@ -45,8 +45,8 @@ ## llm backend implementation ## ======================================================= -## Use oobagooda (default) -#LLM_BACKEND=oobagooda +## Use oobabooga (default) +#LLM_BACKEND=oobabooga #LLM_URL=http://xxx:7860/api/generate/v1 ## Using serverless modal.com (require reflector-gpu-modal deployed) diff --git a/server/reflector/llm/base.py b/server/reflector/llm/base.py index fddf185d..2d83913e 100644 --- a/server/reflector/llm/base.py +++ b/server/reflector/llm/base.py @@ -2,7 +2,6 @@ import importlib import json import re from time import monotonic -from typing import Union from reflector.logger import logger as reflector_logger from reflector.settings import settings @@ -47,7 +46,7 @@ class LLM: pass async def generate( - self, prompt: str, logger: reflector_logger, schema: str = None, **kwargs + self, prompt: str, logger: reflector_logger, schema: str | None = None, **kwargs ) -> dict: logger.info("LLM generate", prompt=repr(prompt)) try: @@ -63,7 +62,7 @@ class LLM: return result - async def _generate(self, prompt: str, schema: Union[str | None], **kwargs) -> str: + async def _generate(self, prompt: str, schema: str | None, **kwargs) -> str: raise NotImplementedError def _parse_json(self, result: str) -> dict: diff --git a/server/reflector/llm/llm_banana.py b/server/reflector/llm/llm_banana.py index 473769cc..e19171bf 100644 --- a/server/reflector/llm/llm_banana.py +++ b/server/reflector/llm/llm_banana.py @@ -1,5 +1,4 @@ import json -from typing import Union import httpx from reflector.llm.base import LLM @@ -16,7 +15,7 @@ class BananaLLM(LLM): "X-Banana-Model-Key": settings.LLM_BANANA_MODEL_KEY, } - async def _generate(self, prompt: str, schema: Union[str | None], **kwargs): + async def _generate(self, prompt: str, schema: str | None, **kwargs): json_payload = {"prompt": prompt} if schema: json_payload["schema"] = json.dumps(schema) diff --git a/server/reflector/llm/llm_modal.py b/server/reflector/llm/llm_modal.py index c1fb856b..7cf7778b 100644 --- a/server/reflector/llm/llm_modal.py +++ b/server/reflector/llm/llm_modal.py @@ -1,5 +1,4 @@ import json -from typing import Union import httpx from reflector.llm.base import LLM @@ -26,7 +25,7 @@ class ModalLLM(LLM): ) response.raise_for_status() - async def _generate(self, prompt: str, schema: Union[str | None], **kwargs): + async def _generate(self, prompt: str, schema: str | None, **kwargs): json_payload = {"prompt": prompt} if schema: json_payload["schema"] = json.dumps(schema) diff --git a/server/reflector/llm/llm_oobagooda.py b/server/reflector/llm/llm_oobabooga.py similarity index 84% rename from server/reflector/llm/llm_oobagooda.py rename to server/reflector/llm/llm_oobabooga.py index 0ceb442d..394f0af4 100644 --- a/server/reflector/llm/llm_oobagooda.py +++ b/server/reflector/llm/llm_oobabooga.py @@ -1,5 +1,4 @@ import json -from typing import Union import httpx from reflector.llm.base import LLM @@ -7,7 +6,7 @@ from reflector.settings import settings class OobaboogaLLM(LLM): - async def _generate(self, prompt: str, schema: Union[str | None], **kwargs): + async def _generate(self, prompt: str, schema: str | None, **kwargs): json_payload = {"prompt": prompt} if schema: json_payload["schema"] = json.dumps(schema) diff --git a/server/reflector/llm/llm_openai.py b/server/reflector/llm/llm_openai.py index 9a74e03c..62cccf7e 100644 --- a/server/reflector/llm/llm_openai.py +++ b/server/reflector/llm/llm_openai.py @@ -1,5 +1,3 @@ -from typing import Union - import httpx from reflector.llm.base import LLM from reflector.logger import logger @@ -17,7 +15,7 @@ class OpenAILLM(LLM): self.max_tokens = settings.LLM_MAX_TOKENS logger.info(f"LLM use openai backend at {self.openai_url}") - async def _generate(self, prompt: str, schema: Union[str | None], **kwargs) -> str: + async def _generate(self, prompt: str, schema: str | None, **kwargs) -> str: headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.openai_key}", diff --git a/server/tests/test_processors_pipeline.py b/server/tests/test_processors_pipeline.py index 56cac96e..db0a39c5 100644 --- a/server/tests/test_processors_pipeline.py +++ b/server/tests/test_processors_pipeline.py @@ -9,16 +9,13 @@ async def test_basic_process(event_loop): from reflector.settings import settings from reflector.llm.base import LLM from pathlib import Path - from typing import Union # use an LLM test backend settings.LLM_BACKEND = "test" settings.TRANSCRIPT_BACKEND = "whisper" class LLMTest(LLM): - async def _generate( - self, prompt: str, schema: Union[str | None], **kwargs - ) -> str: + async def _generate(self, prompt: str, schema: str | None, **kwargs) -> str: return { "title": "TITLE", "summary": "SUMMARY", diff --git a/server/tests/test_transcripts_rtc_ws.py b/server/tests/test_transcripts_rtc_ws.py index 5555d195..943955f8 100644 --- a/server/tests/test_transcripts_rtc_ws.py +++ b/server/tests/test_transcripts_rtc_ws.py @@ -7,7 +7,6 @@ import asyncio import json import threading from pathlib import Path -from typing import Union from unittest.mock import patch import pytest @@ -62,7 +61,7 @@ async def dummy_llm(): from reflector.llm.base import LLM class TestLLM(LLM): - async def _generate(self, prompt: str, schema: Union[str | None], **kwargs): + async def _generate(self, prompt: str, schema: str | None, **kwargs): return json.dumps({"title": "LLM TITLE", "summary": "LLM SUMMARY"}) with patch("reflector.llm.base.LLM.get_instance") as mock_llm: From 9103c8cca83de1459447aec5b3971b963264dce8 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Thu, 17 Aug 2023 15:15:43 +0530 Subject: [PATCH 13/31] remove ast --- server/gpu/modal/reflector_llm.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/gpu/modal/reflector_llm.py b/server/gpu/modal/reflector_llm.py index 21306763..10cf4772 100644 --- a/server/gpu/modal/reflector_llm.py +++ b/server/gpu/modal/reflector_llm.py @@ -3,11 +3,11 @@ Reflector GPU backend - LLM =========================== """ - +import json import os from typing import Optional -from modal import asgi_app, Image, method, Secret, Stub +from modal import Image, Secret, Stub, asgi_app, method # LLM LLM_MODEL: str = "lmsys/vicuna-13b-v1.5" @@ -116,12 +116,11 @@ class LLM: # If a schema is given, conform to schema if schema: print(f"Schema {schema=}") - import ast import jsonformer jsonformer_llm = jsonformer.Jsonformer(model=self.model, tokenizer=self.tokenizer, - json_schema=ast.literal_eval(schema), + json_schema=json.loads(schema), prompt=prompt, max_string_token_length=self.gen_cfg.max_new_tokens) response = jsonformer_llm() @@ -154,7 +153,7 @@ class LLM: ) @asgi_app() def web(): - from fastapi import FastAPI, HTTPException, status, Depends + from fastapi import Depends, FastAPI, HTTPException, status from fastapi.security import OAuth2PasswordBearer from pydantic import BaseModel From 4e9940fe2920d0ba637c2ffc41f61b0a604b03f8 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Thu, 17 Aug 2023 14:46:48 +0200 Subject: [PATCH 14/31] server: fixes tests --- server/reflector/auth/auth_none.py | 17 ++++------------- server/reflector/views/transcripts.py | 3 +++ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/server/reflector/auth/auth_none.py b/server/reflector/auth/auth_none.py index 3959c739..1c1dd0fd 100644 --- a/server/reflector/auth/auth_none.py +++ b/server/reflector/auth/auth_none.py @@ -3,7 +3,7 @@ from typing import Annotated from fastapi import Depends from fastapi.security import OAuth2PasswordBearer -oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") +oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token", auto_error=False) class UserInfo(BaseModel): @@ -15,21 +15,12 @@ class AccessTokenInfo(BaseModel): def authenticated(token: Annotated[str, Depends(oauth2_scheme)]): - def _authenticated(): - return None - - return _authenticated + return None def current_user(token: Annotated[str, Depends(oauth2_scheme)]): - def _current_user(): - return None - - return _current_user + return None def current_user_optional(token: Annotated[str, Depends(oauth2_scheme)]): - def _current_user_optional(): - return None - - return _current_user_optional + return None diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 778c47d7..532de6ff 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -217,6 +217,9 @@ class DeletionStatus(BaseModel): async def transcripts_list( user: auth.UserInfo = Depends(auth.current_user), ): + if not user: + raise HTTPException(status_code=401, detail="Not authenticated") + return paginate(await transcripts_controller.get_all(user_id=user["sub"])) From ca4a4d24324474c086f76e7a3d010c498f9c6e6a Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 17 Aug 2023 21:36:48 +0700 Subject: [PATCH 15/31] Update dashboard.tsx --- www/app/transcripts/dashboard.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/app/transcripts/dashboard.tsx b/www/app/transcripts/dashboard.tsx index 6fc2cfbe..9eb05536 100644 --- a/www/app/transcripts/dashboard.tsx +++ b/www/app/transcripts/dashboard.tsx @@ -98,7 +98,7 @@ export function Dashboard({ )} - {finalSummary && } + {finalSummary.summary && } {disconnected && } From b43bd00fc0cc8a53087d54dd21ec57aaaa12c970 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Thu, 17 Aug 2023 16:49:22 +0200 Subject: [PATCH 16/31] server: fixes wav not saved correctly and mp3 generation invalid if started from /tmp from another device --- server/.gitignore | 1 + server/reflector/processors/audio_file_writer.py | 6 +++--- server/reflector/views/transcripts.py | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/server/.gitignore b/server/.gitignore index 7d66d6f0..dbabe979 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -178,3 +178,4 @@ audio_*.wav # ignore local database reflector.sqlite3 +data/ diff --git a/server/reflector/processors/audio_file_writer.py b/server/reflector/processors/audio_file_writer.py index d67db65e..00ab2529 100644 --- a/server/reflector/processors/audio_file_writer.py +++ b/server/reflector/processors/audio_file_writer.py @@ -26,13 +26,13 @@ class AudioFileWriterProcessor(Processor): self.out_stream = self.out_container.add_stream( "pcm_s16le", rate=data.sample_rate ) - for packet in self.out_stream.encode(data): - self.out_container.mux(packet) + for packet in self.out_stream.encode(data): + self.out_container.mux(packet) await self.emit(data) async def _flush(self): if self.out_container: - for packet in self.out_stream.encode(None): + for packet in self.out_stream.encode(): self.out_container.mux(packet) self.out_container.close() self.out_container = None diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 6f952938..beaafb76 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -108,7 +108,9 @@ class Transcript(BaseModel): out.close() # move temporary file to final location - Path(tmp.name).rename(fn) + import shutil + + shutil.move(tmp.name, fn.as_posix()) def unlink(self): self.data_path.unlink(missing_ok=True) From b08724a191df2964db437e6d1b1a4217e0808064 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Thu, 17 Aug 2023 20:57:31 +0530 Subject: [PATCH 17/31] correct schema typing from str to dict --- server/reflector/llm/base.py | 8 ++++++-- server/reflector/llm/llm_banana.py | 2 +- server/reflector/llm/llm_modal.py | 2 +- server/reflector/llm/llm_oobabooga.py | 2 +- server/reflector/llm/llm_openai.py | 2 +- server/tests/test_processors_pipeline.py | 2 +- server/tests/test_transcripts_rtc_ws.py | 2 +- 7 files changed, 12 insertions(+), 8 deletions(-) diff --git a/server/reflector/llm/base.py b/server/reflector/llm/base.py index 2d83913e..d046ffe7 100644 --- a/server/reflector/llm/base.py +++ b/server/reflector/llm/base.py @@ -46,7 +46,11 @@ class LLM: pass async def generate( - self, prompt: str, logger: reflector_logger, schema: str | None = None, **kwargs + self, + prompt: str, + logger: reflector_logger, + schema: dict | None = None, + **kwargs, ) -> dict: logger.info("LLM generate", prompt=repr(prompt)) try: @@ -62,7 +66,7 @@ class LLM: return result - async def _generate(self, prompt: str, schema: str | None, **kwargs) -> str: + async def _generate(self, prompt: str, schema: dict | None, **kwargs) -> str: raise NotImplementedError def _parse_json(self, result: str) -> dict: diff --git a/server/reflector/llm/llm_banana.py b/server/reflector/llm/llm_banana.py index e19171bf..07119613 100644 --- a/server/reflector/llm/llm_banana.py +++ b/server/reflector/llm/llm_banana.py @@ -15,7 +15,7 @@ class BananaLLM(LLM): "X-Banana-Model-Key": settings.LLM_BANANA_MODEL_KEY, } - async def _generate(self, prompt: str, schema: str | None, **kwargs): + async def _generate(self, prompt: str, schema: dict | None, **kwargs): json_payload = {"prompt": prompt} if schema: json_payload["schema"] = json.dumps(schema) diff --git a/server/reflector/llm/llm_modal.py b/server/reflector/llm/llm_modal.py index 7cf7778b..ea9ff152 100644 --- a/server/reflector/llm/llm_modal.py +++ b/server/reflector/llm/llm_modal.py @@ -25,7 +25,7 @@ class ModalLLM(LLM): ) response.raise_for_status() - async def _generate(self, prompt: str, schema: str | None, **kwargs): + async def _generate(self, prompt: str, schema: dict | None, **kwargs): json_payload = {"prompt": prompt} if schema: json_payload["schema"] = json.dumps(schema) diff --git a/server/reflector/llm/llm_oobabooga.py b/server/reflector/llm/llm_oobabooga.py index 394f0af4..6c5a68ec 100644 --- a/server/reflector/llm/llm_oobabooga.py +++ b/server/reflector/llm/llm_oobabooga.py @@ -6,7 +6,7 @@ from reflector.settings import settings class OobaboogaLLM(LLM): - async def _generate(self, prompt: str, schema: str | None, **kwargs): + async def _generate(self, prompt: str, schema: dict | None, **kwargs): json_payload = {"prompt": prompt} if schema: json_payload["schema"] = json.dumps(schema) diff --git a/server/reflector/llm/llm_openai.py b/server/reflector/llm/llm_openai.py index 62cccf7e..7ed532b7 100644 --- a/server/reflector/llm/llm_openai.py +++ b/server/reflector/llm/llm_openai.py @@ -15,7 +15,7 @@ class OpenAILLM(LLM): self.max_tokens = settings.LLM_MAX_TOKENS logger.info(f"LLM use openai backend at {self.openai_url}") - async def _generate(self, prompt: str, schema: str | None, **kwargs) -> str: + async def _generate(self, prompt: str, schema: dict | None, **kwargs) -> str: headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.openai_key}", diff --git a/server/tests/test_processors_pipeline.py b/server/tests/test_processors_pipeline.py index db0a39c5..cc6a8574 100644 --- a/server/tests/test_processors_pipeline.py +++ b/server/tests/test_processors_pipeline.py @@ -15,7 +15,7 @@ async def test_basic_process(event_loop): settings.TRANSCRIPT_BACKEND = "whisper" class LLMTest(LLM): - async def _generate(self, prompt: str, schema: str | None, **kwargs) -> str: + async def _generate(self, prompt: str, schema: dict | None, **kwargs) -> str: return { "title": "TITLE", "summary": "SUMMARY", diff --git a/server/tests/test_transcripts_rtc_ws.py b/server/tests/test_transcripts_rtc_ws.py index 943955f8..09f3d7e5 100644 --- a/server/tests/test_transcripts_rtc_ws.py +++ b/server/tests/test_transcripts_rtc_ws.py @@ -61,7 +61,7 @@ async def dummy_llm(): from reflector.llm.base import LLM class TestLLM(LLM): - async def _generate(self, prompt: str, schema: str | None, **kwargs): + async def _generate(self, prompt: str, schema: dict | None, **kwargs): return json.dumps({"title": "LLM TITLE", "summary": "LLM SUMMARY"}) with patch("reflector.llm.base.LLM.get_instance") as mock_llm: From 1b1e67901c098321dcf11e6869c77be9fb2d36ed Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 17 Aug 2023 22:57:59 +0700 Subject: [PATCH 18/31] FIEF NextJS --- www/app/layout.tsx | 15 ++++++++---- www/app/lib/fief.ts | 46 +++++++++++++++++++++++++++++++++++ www/middleware.ts | 20 +++++++++++++++ www/package.json | 1 + www/pages/api/current-user.ts | 3 +++ www/pages/forbidden.tsx | 7 ++++++ www/yarn.lock | 36 ++++++++++++++++++++++++++- 7 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 www/app/lib/fief.ts create mode 100644 www/middleware.ts create mode 100644 www/pages/api/current-user.ts create mode 100644 www/pages/forbidden.tsx diff --git a/www/app/layout.tsx b/www/app/layout.tsx index 99733717..e98b2cc4 100644 --- a/www/app/layout.tsx +++ b/www/app/layout.tsx @@ -1,6 +1,7 @@ import "./styles/globals.scss"; import { Roboto } from "next/font/google"; import { Metadata } from "next"; +import { FiefAuthProvider } from "@fief/fief/nextjs/react"; const roboto = Roboto({ subsets: ["latin"], weight: "400" }); @@ -50,10 +51,14 @@ export const metadata: Metadata = { export default function RootLayout({ children }) { return ( - - - {children} - - + + {" "} + {} + + + {children} + + + ); } diff --git a/www/app/lib/fief.ts b/www/app/lib/fief.ts new file mode 100644 index 00000000..4b83a03b --- /dev/null +++ b/www/app/lib/fief.ts @@ -0,0 +1,46 @@ +import { Fief, FiefUserInfo } from "@fief/fief"; +import { FiefAuth, IUserInfoCache } from "@fief/fief/nextjs"; + +export const SESSION_COOKIE_NAME = "reflector-auth"; + +export const fiefClient = new Fief({ + baseURL: process.env.FIEF_URL ?? "", + clientId: process.env.FIEF_CLIENT_ID ?? "", + clientSecret: process.env.FIEF_CLIENT_SECRET ?? "", +}); + +class MemoryUserInfoCache implements IUserInfoCache { + private storage: Record; + + constructor() { + this.storage = {}; + } + + async get(id: string): Promise { + const userinfo = this.storage[id]; + if (userinfo) { + return userinfo; + } + return null; + } + + async set(id: string, userinfo: FiefUserInfo): Promise { + this.storage[id] = userinfo; + } + + async remove(id: string): Promise { + this.storage[id] = undefined; + } + + async clear(): Promise { + this.storage = {}; + } +} + +export const fiefAuth = new FiefAuth({ + client: fiefClient, + sessionCookieName: SESSION_COOKIE_NAME, + redirectURI: "http://localhost:3000/auth-callback", + logoutRedirectURI: "http://localhost:3000", + userInfoCache: new MemoryUserInfoCache(), +}); diff --git a/www/middleware.ts b/www/middleware.ts new file mode 100644 index 00000000..f65a3a67 --- /dev/null +++ b/www/middleware.ts @@ -0,0 +1,20 @@ +import type { NextRequest } from "next/server"; + +import { fiefAuth } from "./app/lib/fief"; + +const authMiddleware = fiefAuth.middleware([ + { + matcher: "/private", + parameters: {}, + }, + { + matcher: "/castles/:path*", + parameters: { + permissions: ["castles:read"], + }, + }, +]); + +export async function middleware(request: NextRequest) { + return authMiddleware(request); +} diff --git a/www/package.json b/www/package.json index ee03840a..2173ce5a 100644 --- a/www/package.json +++ b/www/package.json @@ -11,6 +11,7 @@ "openapi": "openapi-generator-cli generate -i http://localhost:1250/openapi.json -g typescript-fetch -o app/api && yarn format" }, "dependencies": { + "@fief/fief": "^0.13.5", "@fortawesome/fontawesome-svg-core": "^6.4.0", "@fortawesome/free-solid-svg-icons": "^6.4.0", "@fortawesome/react-fontawesome": "^0.2.0", diff --git a/www/pages/api/current-user.ts b/www/pages/api/current-user.ts new file mode 100644 index 00000000..ccb53362 --- /dev/null +++ b/www/pages/api/current-user.ts @@ -0,0 +1,3 @@ +import { fiefAuth } from "../../app/lib/fief"; + +export default fiefAuth.currentUser(); diff --git a/www/pages/forbidden.tsx b/www/pages/forbidden.tsx new file mode 100644 index 00000000..31a746fc --- /dev/null +++ b/www/pages/forbidden.tsx @@ -0,0 +1,7 @@ +import type { NextPage } from "next"; + +const Forbidden: NextPage = () => { + return

Sorry, you are not authorized to access this page.

; +}; + +export default Forbidden; diff --git a/www/yarn.lock b/www/yarn.lock index d2ae9035..dc6ec909 100644 --- a/www/yarn.lock +++ b/www/yarn.lock @@ -14,6 +14,16 @@ dependencies: regenerator-runtime "^0.14.0" +"@fief/fief@^0.13.5": + version "0.13.5" + resolved "https://registry.yarnpkg.com/@fief/fief/-/fief-0.13.5.tgz#01ac833ddff0b84f2e1737cc168721568b0e7a39" + integrity sha512-st4+/Rc1rw18B6RtqLmC6t9k0pnYLG7AqMe0JYjKYcamCNnABwl198ZJt6HShtaZKI5maHsh99UoCA3MqpbWsQ== + dependencies: + encoding "^0.1.13" + jose "^4.6.0" + node-fetch "^2.6.7" + path-to-regexp "^6.2.1" + "@fortawesome/fontawesome-common-types@6.4.0": version "6.4.0" resolved "https://registry.npmjs.org/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.4.0.tgz" @@ -868,6 +878,13 @@ emoji-regex@^8.0.0: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== +encoding@^0.1.13: + version "0.1.13" + resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9" + integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A== + dependencies: + iconv-lite "^0.6.2" + err-code@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/err-code/-/err-code-3.0.1.tgz" @@ -1108,6 +1125,13 @@ iconv-lite@^0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" +iconv-lite@^0.6.2: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + ieee754@^1.1.13, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz" @@ -1247,6 +1271,11 @@ jiti@^1.18.2: resolved "https://registry.npmjs.org/jiti/-/jiti-1.19.1.tgz" integrity sha512-oVhqoRDaBXf7sjkll95LHVS6Myyyb1zaunVwk4Z0+WPSW4gjS0pl01zYKHScTuyEhQsFxV5L4DR5r+YqSyqyyg== +jose@^4.6.0: + version "4.14.4" + resolved "https://registry.yarnpkg.com/jose/-/jose-4.14.4.tgz#59e09204e2670c3164ee24cbfe7115c6f8bff9ca" + integrity sha512-j8GhLiKmUAh+dsFXlX1aJCbt5KMibuKb+d7j1JaOJG6s2UjX1PQlW+OKB/sD4a/5ZYF4RcmYmLSndOoU3Lt/3g== + "js-tokens@^3.0.0 || ^4.0.0": version "4.0.0" resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz" @@ -1529,6 +1558,11 @@ path-to-regexp@3.2.0: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-3.2.0.tgz#fa7877ecbc495c601907562222453c43cc204a5f" integrity sha512-jczvQbCUS7XmS7o+y1aEO9OBVFeZBQ1MDSEqmO7xSoPgOPoowY/SxLpZ6Vh97/8qHZOteiCKb7gkG9gA2ZUxJA== +path-to-regexp@^6.2.1: + version "6.2.1" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.2.1.tgz#d54934d6798eb9e5ef14e7af7962c945906918e5" + integrity sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw== + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz" @@ -1786,7 +1820,7 @@ safe-buffer@^5.1.0, safe-buffer@~5.2.0: resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -"safer-buffer@>= 2.1.2 < 3": +"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== From 2d686da15c98e83731602dbc878e67633baf0316 Mon Sep 17 00:00:00 2001 From: Gokul Mohanarangan Date: Thu, 17 Aug 2023 21:26:20 +0530 Subject: [PATCH 19/31] pass schema as dict --- server/gpu/modal/reflector_llm.py | 7 +++++-- server/reflector/llm/llm_banana.py | 4 +--- server/reflector/llm/llm_modal.py | 4 +--- server/reflector/llm/llm_oobabooga.py | 4 +--- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/server/gpu/modal/reflector_llm.py b/server/gpu/modal/reflector_llm.py index 10cf4772..fd8a4aae 100644 --- a/server/gpu/modal/reflector_llm.py +++ b/server/gpu/modal/reflector_llm.py @@ -172,13 +172,16 @@ def web(): class LLMRequest(BaseModel): prompt: str - schema: Optional[str] = None + schema: Optional[dict] = None @app.post("/llm", dependencies=[Depends(apikey_auth)]) async def llm( req: LLMRequest, ): - func = llmstub.generate.spawn(prompt=req.prompt, schema=req.schema) + if req.schema: + func = llmstub.generate.spawn(prompt=req.prompt, schema=json.dumps(req.schema)) + else: + func = llmstub.generate.spawn(prompt=req.prompt) result = func.get() return result diff --git a/server/reflector/llm/llm_banana.py b/server/reflector/llm/llm_banana.py index 07119613..56fc0e69 100644 --- a/server/reflector/llm/llm_banana.py +++ b/server/reflector/llm/llm_banana.py @@ -1,5 +1,3 @@ -import json - import httpx from reflector.llm.base import LLM from reflector.settings import settings @@ -18,7 +16,7 @@ class BananaLLM(LLM): async def _generate(self, prompt: str, schema: dict | None, **kwargs): json_payload = {"prompt": prompt} if schema: - json_payload["schema"] = json.dumps(schema) + json_payload["schema"] = schema async with httpx.AsyncClient() as client: response = await retry(client.post)( settings.LLM_URL, diff --git a/server/reflector/llm/llm_modal.py b/server/reflector/llm/llm_modal.py index ea9ff152..ce0de02a 100644 --- a/server/reflector/llm/llm_modal.py +++ b/server/reflector/llm/llm_modal.py @@ -1,5 +1,3 @@ -import json - import httpx from reflector.llm.base import LLM from reflector.settings import settings @@ -28,7 +26,7 @@ class ModalLLM(LLM): async def _generate(self, prompt: str, schema: dict | None, **kwargs): json_payload = {"prompt": prompt} if schema: - json_payload["schema"] = json.dumps(schema) + json_payload["schema"] = schema async with httpx.AsyncClient() as client: response = await retry(client.post)( self.llm_url, diff --git a/server/reflector/llm/llm_oobabooga.py b/server/reflector/llm/llm_oobabooga.py index 6c5a68ec..411014c5 100644 --- a/server/reflector/llm/llm_oobabooga.py +++ b/server/reflector/llm/llm_oobabooga.py @@ -1,5 +1,3 @@ -import json - import httpx from reflector.llm.base import LLM from reflector.settings import settings @@ -9,7 +7,7 @@ class OobaboogaLLM(LLM): async def _generate(self, prompt: str, schema: dict | None, **kwargs): json_payload = {"prompt": prompt} if schema: - json_payload["schema"] = json.dumps(schema) + json_payload["schema"] = schema async with httpx.AsyncClient() as client: response = await client.post( settings.LLM_URL, From 067d1fbd4b7f50d49baa6a118bf34333d51a9f55 Mon Sep 17 00:00:00 2001 From: Koper Date: Thu, 17 Aug 2023 23:32:42 +0700 Subject: [PATCH 20/31] NextJS Fief Auth (Draft) --- www/app/api/apis/DefaultApi.ts | 99 +++++++++++++++++++++++++++ www/app/fiefWrapper.tsx | 11 +++ www/app/layout.tsx | 18 ++--- www/app/lib/fief.ts | 1 + www/app/transcripts/useTranscript.tsx | 1 + www/app/userInfo.tsx | 23 +++++++ 6 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 www/app/fiefWrapper.tsx create mode 100644 www/app/userInfo.tsx diff --git a/www/app/api/apis/DefaultApi.ts b/www/app/api/apis/DefaultApi.ts index 147fd770..752ef536 100644 --- a/www/app/api/apis/DefaultApi.ts +++ b/www/app/api/apis/DefaultApi.ts @@ -55,6 +55,10 @@ export interface V1TranscriptGetAudioRequest { transcriptId: any; } +export interface V1TranscriptGetAudioMp3Request { + transcriptId: any; +} + export interface V1TranscriptGetTopicsRequest { transcriptId: any; } @@ -159,6 +163,14 @@ export class DefaultApi extends runtime.BaseAPI { const headerParameters: runtime.HTTPHeaders = {}; + if (this.configuration && this.configuration.accessToken) { + // oauth required + headerParameters["Authorization"] = await this.configuration.accessToken( + "OAuth2AuthorizationCodeBearer", + [], + ); + } + const response = await this.request( { path: `/v1/transcripts/{transcript_id}`.replace( @@ -212,6 +224,14 @@ export class DefaultApi extends runtime.BaseAPI { const headerParameters: runtime.HTTPHeaders = {}; + if (this.configuration && this.configuration.accessToken) { + // oauth required + headerParameters["Authorization"] = await this.configuration.accessToken( + "OAuth2AuthorizationCodeBearer", + [], + ); + } + const response = await this.request( { path: `/v1/transcripts/{transcript_id}`.replace( @@ -299,6 +319,61 @@ export class DefaultApi extends runtime.BaseAPI { return await response.value(); } + /** + * Transcript Get Audio Mp3 + */ + async v1TranscriptGetAudioMp3Raw( + requestParameters: V1TranscriptGetAudioMp3Request, + initOverrides?: RequestInit | runtime.InitOverrideFunction, + ): Promise> { + if ( + requestParameters.transcriptId === null || + requestParameters.transcriptId === undefined + ) { + throw new runtime.RequiredError( + "transcriptId", + "Required parameter requestParameters.transcriptId was null or undefined when calling v1TranscriptGetAudioMp3.", + ); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + const response = await this.request( + { + path: `/v1/transcripts/{transcript_id}/audio/mp3`.replace( + `{${"transcript_id"}}`, + encodeURIComponent(String(requestParameters.transcriptId)), + ), + method: "GET", + headers: headerParameters, + query: queryParameters, + }, + initOverrides, + ); + + if (this.isJsonMime(response.headers.get("content-type"))) { + return new runtime.JSONApiResponse(response); + } else { + return new runtime.TextApiResponse(response) as any; + } + } + + /** + * Transcript Get Audio Mp3 + */ + async v1TranscriptGetAudioMp3( + requestParameters: V1TranscriptGetAudioMp3Request, + initOverrides?: RequestInit | runtime.InitOverrideFunction, + ): Promise { + const response = await this.v1TranscriptGetAudioMp3Raw( + requestParameters, + initOverrides, + ); + return await response.value(); + } + /** * Transcript Get Topics */ @@ -510,6 +585,14 @@ export class DefaultApi extends runtime.BaseAPI { headerParameters["Content-Type"] = "application/json"; + if (this.configuration && this.configuration.accessToken) { + // oauth required + headerParameters["Authorization"] = await this.configuration.accessToken( + "OAuth2AuthorizationCodeBearer", + [], + ); + } + const response = await this.request( { path: `/v1/transcripts/{transcript_id}`.replace( @@ -566,6 +649,14 @@ export class DefaultApi extends runtime.BaseAPI { headerParameters["Content-Type"] = "application/json"; + if (this.configuration && this.configuration.accessToken) { + // oauth required + headerParameters["Authorization"] = await this.configuration.accessToken( + "OAuth2AuthorizationCodeBearer", + [], + ); + } + const response = await this.request( { path: `/v1/transcripts`, @@ -615,6 +706,14 @@ export class DefaultApi extends runtime.BaseAPI { const headerParameters: runtime.HTTPHeaders = {}; + if (this.configuration && this.configuration.accessToken) { + // oauth required + headerParameters["Authorization"] = await this.configuration.accessToken( + "OAuth2AuthorizationCodeBearer", + [], + ); + } + const response = await this.request( { path: `/v1/transcripts`, diff --git a/www/app/fiefWrapper.tsx b/www/app/fiefWrapper.tsx new file mode 100644 index 00000000..187fef7c --- /dev/null +++ b/www/app/fiefWrapper.tsx @@ -0,0 +1,11 @@ +"use client"; + +import { FiefAuthProvider } from "@fief/fief/nextjs/react"; + +export default function FiefWrapper({ children }) { + return ( + + {children} + + ); +} diff --git a/www/app/layout.tsx b/www/app/layout.tsx index e98b2cc4..85a8f17b 100644 --- a/www/app/layout.tsx +++ b/www/app/layout.tsx @@ -1,7 +1,8 @@ import "./styles/globals.scss"; import { Roboto } from "next/font/google"; import { Metadata } from "next"; -import { FiefAuthProvider } from "@fief/fief/nextjs/react"; +import FiefWrapper from "./fiefWrapper"; +import UserInfo from "./userInfo"; const roboto = Roboto({ subsets: ["latin"], weight: "400" }); @@ -51,14 +52,13 @@ export const metadata: Metadata = { export default function RootLayout({ children }) { return ( - - {" "} - {} - - + + + + {children} - - - + + + ); } diff --git a/www/app/lib/fief.ts b/www/app/lib/fief.ts index 4b83a03b..a2e610b2 100644 --- a/www/app/lib/fief.ts +++ b/www/app/lib/fief.ts @@ -1,3 +1,4 @@ +"use client"; import { Fief, FiefUserInfo } from "@fief/fief"; import { FiefAuth, IUserInfoCache } from "@fief/fief/nextjs"; diff --git a/www/app/transcripts/useTranscript.tsx b/www/app/transcripts/useTranscript.tsx index 11510020..26e0d440 100644 --- a/www/app/transcripts/useTranscript.tsx +++ b/www/app/transcripts/useTranscript.tsx @@ -17,6 +17,7 @@ const useTranscript = (): UseTranscript => { const apiConfiguration = new Configuration({ basePath: process.env.NEXT_PUBLIC_API_URL, + // accessToken: }); const api = new DefaultApi(apiConfiguration); diff --git a/www/app/userInfo.tsx b/www/app/userInfo.tsx new file mode 100644 index 00000000..bf8d19c5 --- /dev/null +++ b/www/app/userInfo.tsx @@ -0,0 +1,23 @@ +"use client"; +import { useFiefUserinfo } from "@fief/fief/nextjs/react"; + +export default function UserInfo() { + const userinfo = useFiefUserinfo(); + + return ( + <> + {userinfo && ( + <> +

Logged In

+

{userinfo.email}

+ + )} + + {!userinfo && ( + <> +

Not Logged In

+ + )} + + ); +} From 7809b60011eeae819df805f9219e40dd93f739a2 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Fri, 18 Aug 2023 10:08:27 +0200 Subject: [PATCH 21/31] server: remove print() statements --- server/reflector/views/transcripts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index bfe61473..29db9ec7 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -136,9 +136,7 @@ class TranscriptController: query = transcripts.select() if user_id is not None: query = query.where(transcripts.c.user_id == user_id) - print(query) results = await database.fetch_all(query) - print(results) return results async def get_by_id( From 5c9adb2664e1f053bf6f6e0597c858af745ae6e9 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Fri, 18 Aug 2023 10:23:15 +0200 Subject: [PATCH 22/31] server: fixes tests --- server/tests/test_transcripts_rtc_ws.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/tests/test_transcripts_rtc_ws.py b/server/tests/test_transcripts_rtc_ws.py index 28425cd5..8237d4ab 100644 --- a/server/tests/test_transcripts_rtc_ws.py +++ b/server/tests/test_transcripts_rtc_ws.py @@ -12,7 +12,6 @@ from unittest.mock import patch import pytest from httpx import AsyncClient from httpx_ws import aconnect_ws -from reflector.app import app from uvicorn import Config, Server From 2a3ad5657f75fa9de0bf57928c524f4f7a0a055c Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Fri, 18 Aug 2023 12:02:16 +0200 Subject: [PATCH 23/31] server: add /v1/me to get current user information sub, email and email_verified --- server/reflector/app.py | 17 ++++++++++------- server/reflector/views/user.py | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 server/reflector/views/user.py diff --git a/server/reflector/app.py b/server/reflector/app.py index fa148240..14d55d7a 100644 --- a/server/reflector/app.py +++ b/server/reflector/app.py @@ -1,15 +1,17 @@ +from contextlib import asynccontextmanager + +import reflector.auth # noqa +import reflector.db # noqa from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi_pagination import add_pagination from fastapi.routing import APIRoute -import reflector.db # noqa -import reflector.auth # noqa -from reflector.views.rtc_offer import router as rtc_offer_router -from reflector.views.transcripts import router as transcripts_router -from reflector.events import subscribers_startup, subscribers_shutdown +from fastapi_pagination import add_pagination +from reflector.events import subscribers_shutdown, subscribers_startup from reflector.logger import logger from reflector.settings import settings -from contextlib import asynccontextmanager +from reflector.views.rtc_offer import router as rtc_offer_router +from reflector.views.transcripts import router as transcripts_router +from reflector.views.user import router as user_router try: import sentry_sdk @@ -50,6 +52,7 @@ app.add_middleware( # register views app.include_router(rtc_offer_router) app.include_router(transcripts_router, prefix="/v1") +app.include_router(user_router, prefix="/v1") add_pagination(app) diff --git a/server/reflector/views/user.py b/server/reflector/views/user.py new file mode 100644 index 00000000..4952c471 --- /dev/null +++ b/server/reflector/views/user.py @@ -0,0 +1,20 @@ +from typing import Annotated, Optional + +import reflector.auth as auth +from fastapi import APIRouter, Depends +from pydantic import BaseModel + +router = APIRouter() + + +class UserInfo(BaseModel): + sub: str + email: Optional[str] + email_verified: Optional[bool] + + +@router.get("/me") +async def user_me( + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +) -> UserInfo | None: + return user From 0c93a39e337534f2bd1472d1e363d7d5df7c7b8a Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Fri, 18 Aug 2023 12:39:19 +0200 Subject: [PATCH 24/31] server: add transcript verification on get_by_id --- .pre-commit-config.yaml | 1 + server/reflector/views/transcripts.py | 67 ++++++++++++++++----------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a73b7c2..3dcbe202 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,3 +36,4 @@ repos: - id: isort name: isort (python) files: ^server/(gpu|evaluate|reflector)/ + args: ["--profile", "black", "--filter-files"] diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 29db9ec7..9a5c7dfe 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -1,27 +1,28 @@ +from datetime import datetime +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Annotated, Optional +from uuid import uuid4 + +import av +import reflector.auth as auth from fastapi import ( APIRouter, + Depends, HTTPException, Request, WebSocket, WebSocketDisconnect, - Depends, ) from fastapi.responses import FileResponse -from starlette.concurrency import run_in_threadpool -from pydantic import BaseModel, Field -from uuid import uuid4 -from datetime import datetime from fastapi_pagination import Page, paginate -from reflector.logger import logger +from pydantic import BaseModel, Field from reflector.db import database, transcripts +from reflector.logger import logger from reflector.settings import settings -import reflector.auth as auth -from .rtc_offer import rtc_offer_base, RtcOffer, PipelineEvent -from typing import Annotated, Optional -from pathlib import Path -from tempfile import NamedTemporaryFile -import av +from starlette.concurrency import run_in_threadpool +from .rtc_offer import PipelineEvent, RtcOffer, rtc_offer_base router = APIRouter() @@ -139,15 +140,13 @@ class TranscriptController: results = await database.fetch_all(query) return results - async def get_by_id( - self, transcript_id: str, user_id: str | None = None - ) -> Transcript | None: + async def get_by_id(self, transcript_id: str, **kwargs) -> Transcript | None: query = transcripts.select().where(transcripts.c.id == transcript_id) + if "user_id" in kwargs: + query = query.where(transcripts.c.user_id == kwargs["user_id"]) result = await database.fetch_one(query) if not result: return None - if user_id is not None and result["user_id"] != user_id: - return None return Transcript(**result) async def add(self, name: str, user_id: str | None = None): @@ -169,7 +168,7 @@ class TranscriptController: async def remove_by_id( self, transcript_id: str, user_id: str | None = None ) -> None: - transcript = await self.get_by_id(transcript_id) + transcript = await self.get_by_id(transcript_id, user_id=user_id) if not transcript: return if user_id is not None and transcript.user_id != user_id: @@ -282,8 +281,12 @@ async def transcript_delete( @router.get("/transcripts/{transcript_id}/audio") -async def transcript_get_audio(transcript_id: str): - transcript = await transcripts_controller.get_by_id(transcript_id) +async def transcript_get_audio( + transcript_id: str, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +): + user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") @@ -294,8 +297,12 @@ async def transcript_get_audio(transcript_id: str): @router.get("/transcripts/{transcript_id}/audio/mp3") -async def transcript_get_audio_mp3(transcript_id: str): - transcript = await transcripts_controller.get_by_id(transcript_id) +async def transcript_get_audio_mp3( + transcript_id: str, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +): + user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") @@ -308,8 +315,12 @@ async def transcript_get_audio_mp3(transcript_id: str): @router.get("/transcripts/{transcript_id}/topics", response_model=list[TranscriptTopic]) -async def transcript_get_topics(transcript_id: str): - transcript = await transcripts_controller.get_by_id(transcript_id) +async def transcript_get_topics( + transcript_id: str, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +): + user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") return transcript.topics @@ -457,9 +468,13 @@ async def handle_rtc_event(event: PipelineEvent, args, data): @router.post("/transcripts/{transcript_id}/record/webrtc") async def transcript_record_webrtc( - transcript_id: str, params: RtcOffer, request: Request + transcript_id: str, + params: RtcOffer, + request: Request, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], ): - transcript = await transcripts_controller.get_by_id(transcript_id) + user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") From 2339be41725cc537739d4ecb7b8287fdb97c9e65 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Fri, 18 Aug 2023 12:45:59 +0200 Subject: [PATCH 25/31] server: add PUBLIC_MODE settings to allow listing for anonymous user --- server/env.example | 9 +++++++++ server/reflector/settings.py | 4 ++++ server/reflector/views/transcripts.py | 16 +++++++++++----- server/tests/test_transcripts.py | 10 ++++++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/server/env.example b/server/env.example index 0dc73c22..5c91b9d2 100644 --- a/server/env.example +++ b/server/env.example @@ -25,6 +25,15 @@ #AUTH_FIEF_CLIENT_SECRET=xxx +## ======================================================= +## Public mode +## ======================================================= +## If set to true, anonymous transcripts will be +## accessible to anybody. + +#PUBLIC_MODE=false + + ## ======================================================= ## Transcription backend ## diff --git a/server/reflector/settings.py b/server/reflector/settings.py index 468dab2f..396ce7a3 100644 --- a/server/reflector/settings.py +++ b/server/reflector/settings.py @@ -87,5 +87,9 @@ class Settings(BaseSettings): AUTH_FIEF_CLIENT_ID: str | None = None AUTH_FIEF_CLIENT_SECRET: str | None = None + # API public mode + # if set, all anonymous record will be public + PUBLIC_MODE: bool = False + settings = Settings() diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 9a5c7dfe..5c97d33a 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -214,12 +214,13 @@ class DeletionStatus(BaseModel): @router.get("/transcripts", response_model=Page[GetTranscript]) async def transcripts_list( - user: auth.UserInfo = Depends(auth.current_user), + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], ): - if not user: + if not user and not settings.PUBLIC_MODE: raise HTTPException(status_code=401, detail="Not authenticated") - return paginate(await transcripts_controller.get_all(user_id=user["sub"])) + user_id = user["sub"] if user else None + return paginate(await transcripts_controller.get_all(user_id=user_id)) @router.post("/transcripts", response_model=GetTranscript) @@ -367,8 +368,13 @@ ws_manager = WebsocketManager() @router.websocket("/transcripts/{transcript_id}/events") -async def transcript_events_websocket(transcript_id: str, websocket: WebSocket): - transcript = await transcripts_controller.get_by_id(transcript_id) +async def transcript_events_websocket( + transcript_id: str, + websocket: WebSocket, + user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], +): + user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") diff --git a/server/tests/test_transcripts.py b/server/tests/test_transcripts.py index 6badc27d..768018c6 100644 --- a/server/tests/test_transcripts.py +++ b/server/tests/test_transcripts.py @@ -49,11 +49,21 @@ async def test_transcripts_list_anonymous(): # XXX this test is a bit fragile, as it depends on the storage which # is shared between tests from reflector.app import app + from reflector.settings import settings async with AsyncClient(app=app, base_url="http://test/v1") as ac: response = await ac.get("/transcripts") assert response.status_code == 401 + # if public mode, it should be allowed + try: + settings.PUBLIC_MODE = True + async with AsyncClient(app=app, base_url="http://test/v1") as ac: + response = await ac.get("/transcripts") + assert response.status_code == 200 + finally: + settings.PUBLIC_MODE = False + @pytest.fixture @pytest.mark.asyncio From de38b0047d3e4dabe46f4d8b8c44d9261772c5b3 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Fri, 18 Aug 2023 12:49:02 +0200 Subject: [PATCH 26/31] server: deactivate user auth for ws for now --- server/reflector/views/transcripts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 5c97d33a..75422afe 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -371,10 +371,10 @@ ws_manager = WebsocketManager() async def transcript_events_websocket( transcript_id: str, websocket: WebSocket, - user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], + # user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)], ): - user_id = user["sub"] if user else None - transcript = await transcripts_controller.get_by_id(transcript_id, user_id=user_id) + # user_id = user["sub"] if user else None + transcript = await transcripts_controller.get_by_id(transcript_id) if not transcript: raise HTTPException(status_code=404, detail="Transcript not found") From 733054428e37c95c62566dc13b283e34865674dd Mon Sep 17 00:00:00 2001 From: Koper Date: Fri, 18 Aug 2023 19:29:02 +0700 Subject: [PATCH 27/31] Auth front-end --- www/app/{ => (auth)}/fiefWrapper.tsx | 0 www/app/(auth)/userInfo.tsx | 43 ++++++++++++++ www/app/api/.openapi-generator/FILES | 1 + www/app/api/apis/DefaultApi.ts | 45 ++++++++++++++ www/app/api/models/UserInfo.ts | 84 +++++++++++++++++++++++++++ www/app/api/models/index.ts | 1 + www/app/layout.tsx | 19 ++++-- www/app/lib/getApi.ts | 18 ++++++ www/app/lib/{random.tsx => random.ts} | 0 www/app/lib/{time.tsx => time.ts} | 0 www/app/lib/user.ts | 21 +++++++ www/app/transcripts/new/page.tsx | 9 +-- www/app/transcripts/useTranscript.tsx | 7 +-- www/app/transcripts/useWebRTC.tsx | 6 +- www/app/userInfo.tsx | 23 -------- 15 files changed, 234 insertions(+), 43 deletions(-) rename www/app/{ => (auth)}/fiefWrapper.tsx (100%) create mode 100644 www/app/(auth)/userInfo.tsx create mode 100644 www/app/api/models/UserInfo.ts create mode 100644 www/app/lib/getApi.ts rename www/app/lib/{random.tsx => random.ts} (100%) rename www/app/lib/{time.tsx => time.ts} (100%) create mode 100644 www/app/lib/user.ts delete mode 100644 www/app/userInfo.tsx diff --git a/www/app/fiefWrapper.tsx b/www/app/(auth)/fiefWrapper.tsx similarity index 100% rename from www/app/fiefWrapper.tsx rename to www/app/(auth)/fiefWrapper.tsx diff --git a/www/app/(auth)/userInfo.tsx b/www/app/(auth)/userInfo.tsx new file mode 100644 index 00000000..d0a8ad02 --- /dev/null +++ b/www/app/(auth)/userInfo.tsx @@ -0,0 +1,43 @@ +"use client"; +import { + useFiefIsAuthenticated, + useFiefUserinfo, +} from "@fief/fief/nextjs/react"; +import Link from "next/link"; +import Image from "next/image"; + +export default function UserInfo() { + const isAuthenticated = useFiefIsAuthenticated(); + const userinfo = useFiefUserinfo(); + + return ( +
+ {/* Logo on the left */} + + Reflector + + + {/* Text link on the right */} + {!isAuthenticated && ( + + Log in or create account + + )} + {isAuthenticated && ( + + {userinfo?.email} ( + + Log out + + ) + + )} +
+ ); +} diff --git a/www/app/api/.openapi-generator/FILES b/www/app/api/.openapi-generator/FILES index c6dc9bb8..91888ece 100644 --- a/www/app/api/.openapi-generator/FILES +++ b/www/app/api/.openapi-generator/FILES @@ -9,6 +9,7 @@ models/PageGetTranscript.ts models/RtcOffer.ts models/TranscriptTopic.ts models/UpdateTranscript.ts +models/UserInfo.ts models/ValidationError.ts models/index.ts runtime.ts diff --git a/www/app/api/apis/DefaultApi.ts b/www/app/api/apis/DefaultApi.ts index 752ef536..947c64ed 100644 --- a/www/app/api/apis/DefaultApi.ts +++ b/www/app/api/apis/DefaultApi.ts @@ -742,4 +742,49 @@ export class DefaultApi extends runtime.BaseAPI { ); return await response.value(); } + + /** + * User Me + */ + async v1UserMeRaw( + initOverrides?: RequestInit | runtime.InitOverrideFunction, + ): Promise> { + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + if (this.configuration && this.configuration.accessToken) { + // oauth required + headerParameters["Authorization"] = await this.configuration.accessToken( + "OAuth2AuthorizationCodeBearer", + [], + ); + } + + const response = await this.request( + { + path: `/v1/me`, + method: "GET", + headers: headerParameters, + query: queryParameters, + }, + initOverrides, + ); + + if (this.isJsonMime(response.headers.get("content-type"))) { + return new runtime.JSONApiResponse(response); + } else { + return new runtime.TextApiResponse(response) as any; + } + } + + /** + * User Me + */ + async v1UserMe( + initOverrides?: RequestInit | runtime.InitOverrideFunction, + ): Promise { + const response = await this.v1UserMeRaw(initOverrides); + return await response.value(); + } } diff --git a/www/app/api/models/UserInfo.ts b/www/app/api/models/UserInfo.ts new file mode 100644 index 00000000..65d84951 --- /dev/null +++ b/www/app/api/models/UserInfo.ts @@ -0,0 +1,84 @@ +/* tslint:disable */ +/* eslint-disable */ +/** + * FastAPI + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * The version of the OpenAPI document: 0.1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from "../runtime"; +/** + * + * @export + * @interface UserInfo + */ +export interface UserInfo { + /** + * + * @type {any} + * @memberof UserInfo + */ + sub: any | null; + /** + * + * @type {any} + * @memberof UserInfo + */ + email: any | null; + /** + * + * @type {any} + * @memberof UserInfo + */ + emailVerified: any | null; +} + +/** + * Check if a given object implements the UserInfo interface. + */ +export function instanceOfUserInfo(value: object): boolean { + let isInstance = true; + isInstance = isInstance && "sub" in value; + isInstance = isInstance && "email" in value; + isInstance = isInstance && "emailVerified" in value; + + return isInstance; +} + +export function UserInfoFromJSON(json: any): UserInfo { + return UserInfoFromJSONTyped(json, false); +} + +export function UserInfoFromJSONTyped( + json: any, + ignoreDiscriminator: boolean, +): UserInfo { + if (json === undefined || json === null) { + return json; + } + return { + sub: json["sub"], + email: json["email"], + emailVerified: json["email_verified"], + }; +} + +export function UserInfoToJSON(value?: UserInfo | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + sub: value.sub, + email: value.email, + email_verified: value.emailVerified, + }; +} diff --git a/www/app/api/models/index.ts b/www/app/api/models/index.ts index 7de1ce39..d872c782 100644 --- a/www/app/api/models/index.ts +++ b/www/app/api/models/index.ts @@ -8,4 +8,5 @@ export * from "./PageGetTranscript"; export * from "./RtcOffer"; export * from "./TranscriptTopic"; export * from "./UpdateTranscript"; +export * from "./UserInfo"; export * from "./ValidationError"; diff --git a/www/app/layout.tsx b/www/app/layout.tsx index 85a8f17b..fc7913f5 100644 --- a/www/app/layout.tsx +++ b/www/app/layout.tsx @@ -1,8 +1,8 @@ import "./styles/globals.scss"; import { Roboto } from "next/font/google"; import { Metadata } from "next"; -import FiefWrapper from "./fiefWrapper"; -import UserInfo from "./userInfo"; +import FiefWrapper from "./(auth)/fiefWrapper"; +import UserInfo from "./(auth)/userInfo"; const roboto = Roboto({ subsets: ["latin"], weight: "400" }); @@ -55,8 +55,19 @@ export default function RootLayout({ children }) { - - {children} +
+
+ + +
+

Reflector

+

+ Capture The Signal, Not The Noise +

+
+ {children} +
+
diff --git a/www/app/lib/getApi.ts b/www/app/lib/getApi.ts new file mode 100644 index 00000000..198bfd59 --- /dev/null +++ b/www/app/lib/getApi.ts @@ -0,0 +1,18 @@ +import { Configuration } from "../api/runtime"; +import { DefaultApi } from "../api/apis/DefaultApi"; + +import { useFiefAccessTokenInfo } from "@fief/fief/nextjs/react"; + +export default function getApi(): DefaultApi { + const accessTokenInfo = useFiefAccessTokenInfo(); + + const apiConfiguration = new Configuration({ + basePath: process.env.NEXT_PUBLIC_API_URL, + accessToken: accessTokenInfo + ? "Bearer " + accessTokenInfo.access_token + : undefined, + }); + const api = new DefaultApi(apiConfiguration); + + return api; +} diff --git a/www/app/lib/random.tsx b/www/app/lib/random.ts similarity index 100% rename from www/app/lib/random.tsx rename to www/app/lib/random.ts diff --git a/www/app/lib/time.tsx b/www/app/lib/time.ts similarity index 100% rename from www/app/lib/time.tsx rename to www/app/lib/time.ts diff --git a/www/app/lib/user.ts b/www/app/lib/user.ts new file mode 100644 index 00000000..26740e47 --- /dev/null +++ b/www/app/lib/user.ts @@ -0,0 +1,21 @@ +export async function getCurrentUser(): Promise { + try { + const response = await fetch("/api/current-user"); + + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + + const data = await response.json(); + + // Ensure the data structure is as expected + if (data.userinfo && data.access_token_info) { + return data; + } else { + throw new Error("Unexpected data structure"); + } + } catch (error) { + console.error("Error fetching the user data:", error); + throw error; // or you can return an appropriate fallback or error indicator + } +} diff --git a/www/app/transcripts/new/page.tsx b/www/app/transcripts/new/page.tsx index 80bbe9fe..949b477f 100644 --- a/www/app/transcripts/new/page.tsx +++ b/www/app/transcripts/new/page.tsx @@ -26,12 +26,7 @@ const App = () => { const webSockets = useWebSockets(transcript.response?.id); return ( -
-
-

Reflector

-

Capture The Signal, Not The Noise

-
- + <> { @@ -48,7 +43,7 @@ const App = () => { topics={webSockets.topics} disconnected={disconnected} /> -
+ ); }; diff --git a/www/app/transcripts/useTranscript.tsx b/www/app/transcripts/useTranscript.tsx index 26e0d440..040004ac 100644 --- a/www/app/transcripts/useTranscript.tsx +++ b/www/app/transcripts/useTranscript.tsx @@ -2,6 +2,7 @@ import { useEffect, useState } from "react"; import { DefaultApi, V1TranscriptsCreateRequest } from "../api/apis/DefaultApi"; import { Configuration } from "../api/runtime"; import { GetTranscript } from "../api"; +import getApi from "../lib/getApi"; type UseTranscript = { response: GetTranscript | null; @@ -15,11 +16,7 @@ const useTranscript = (): UseTranscript => { const [loading, setLoading] = useState(false); const [error, setError] = useState(null); - const apiConfiguration = new Configuration({ - basePath: process.env.NEXT_PUBLIC_API_URL, - // accessToken: - }); - const api = new DefaultApi(apiConfiguration); + const api = getApi(); const createTranscript = () => { setLoading(true); diff --git a/www/app/transcripts/useWebRTC.tsx b/www/app/transcripts/useWebRTC.tsx index ea622bb2..2f9f3c25 100644 --- a/www/app/transcripts/useWebRTC.tsx +++ b/www/app/transcripts/useWebRTC.tsx @@ -5,6 +5,7 @@ import { V1TranscriptRecordWebrtcRequest, } from "../api/apis/DefaultApi"; import { Configuration } from "../api/runtime"; +import getApi from "../lib/getApi"; const useWebRTC = ( stream: MediaStream | null, @@ -17,10 +18,7 @@ const useWebRTC = ( return; } - const apiConfiguration = new Configuration({ - basePath: process.env.NEXT_PUBLIC_API_URL, - }); - const api = new DefaultApi(apiConfiguration); + const api = getApi(); let p: Peer = new Peer({ initiator: true, stream: stream }); diff --git a/www/app/userInfo.tsx b/www/app/userInfo.tsx deleted file mode 100644 index bf8d19c5..00000000 --- a/www/app/userInfo.tsx +++ /dev/null @@ -1,23 +0,0 @@ -"use client"; -import { useFiefUserinfo } from "@fief/fief/nextjs/react"; - -export default function UserInfo() { - const userinfo = useFiefUserinfo(); - - return ( - <> - {userinfo && ( - <> -

Logged In

-

{userinfo.email}

- - )} - - {!userinfo && ( - <> -

Not Logged In

- - )} - - ); -} From 37a21c3375d5c4cbae3d7d2646023e9a915c2797 Mon Sep 17 00:00:00 2001 From: Koper Date: Fri, 18 Aug 2023 19:40:25 +0700 Subject: [PATCH 28/31] Update README.md --- www/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/README.md b/www/README.md index f5f87643..c92df561 100644 --- a/www/README.md +++ b/www/README.md @@ -79,7 +79,7 @@ This data is then returned from the `useWebRTC` hook and can be used in your com To generate the TypeScript files from the openapi.json file, make sure the python server is running, then run: ``` -openapi-generator-cli generate -i http://localhost:1250/openapi.json -g typescript-fetch -o app/api +yarn openapi ``` You may need to run `yarn global add @openapitools/openapi-generator-cli` first. You also need a Java runtime installed on your machine. From eac40404a906256c95e56bda4d9cae10581824a7 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Fri, 18 Aug 2023 15:28:32 +0200 Subject: [PATCH 29/31] server: fixes get_all --- server/reflector/views/transcripts.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/reflector/views/transcripts.py b/server/reflector/views/transcripts.py index 75422afe..e542973a 100644 --- a/server/reflector/views/transcripts.py +++ b/server/reflector/views/transcripts.py @@ -134,9 +134,7 @@ class Transcript(BaseModel): class TranscriptController: async def get_all(self, user_id: str | None = None) -> list[Transcript]: - query = transcripts.select() - if user_id is not None: - query = query.where(transcripts.c.user_id == user_id) + query = transcripts.select().where(transcripts.c.user_id == user_id) results = await database.fetch_all(query) return results From 60c6acbe682d4851cfe10fe3e557b6784c15136f Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Fri, 18 Aug 2023 16:02:44 +0200 Subject: [PATCH 30/31] gh: do not auto reploy to ecs, manual dispatch only for now --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 015a4e01..3e06d513 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,6 +1,6 @@ name: Deploy to Amazon ECS -on: [deployment, workflow_dispatch] +on: [workflow_dispatch] env: # 384658522150.dkr.ecr.us-east-1.amazonaws.com/reflector From 218bb9c91febc17e5f5720ae01d42ca3da15572d Mon Sep 17 00:00:00 2001 From: Koper Date: Fri, 18 Aug 2023 21:10:43 +0700 Subject: [PATCH 31/31] Moved URLs to .env --- www/app/lib/fief.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/www/app/lib/fief.ts b/www/app/lib/fief.ts index a2e610b2..d778aad7 100644 --- a/www/app/lib/fief.ts +++ b/www/app/lib/fief.ts @@ -41,7 +41,10 @@ class MemoryUserInfoCache implements IUserInfoCache { export const fiefAuth = new FiefAuth({ client: fiefClient, sessionCookieName: SESSION_COOKIE_NAME, - redirectURI: "http://localhost:3000/auth-callback", - logoutRedirectURI: "http://localhost:3000", + redirectURI: + process.env.NEXT_PUBLIC_AUTH_CALLBACK_URL || + "http://localhost:3000/auth-callback", + logoutRedirectURI: + process.env.NEXT_PUBLIC_SITE_URL || "http://localhost:3000", userInfoCache: new MemoryUserInfoCache(), });