diff --git a/server/gpu/modal/reflector_translator.py b/server/gpu/modal/reflector_translator.py index 2986e002..cc7822fc 100644 --- a/server/gpu/modal/reflector_translator.py +++ b/server/gpu/modal/reflector_translator.py @@ -167,194 +167,196 @@ class Translator: """ # TODO: Enhance with complete list of lang codes seamless_lang_code = { + # Afrikaans + 'af': 'afr', # Amharic - "am": "amh", + 'am': 'amh', # Modern Standard Arabic - "ar": "arb", + 'ar': 'arb', # Moroccan Arabic - # (No 2-letter code) + 'ary': 'ary', # Egyptian Arabic - # (No 2-letter code) + 'arz': 'arz', # Assamese - "as": "asm", + 'as': 'asm', # North Azerbaijani - "az": "azj", + 'az': 'azj', # Belarusian - "be": "bel", + 'be': 'bel', # Bengali - "bn": "ben", + 'bn': 'ben', # Bosnian - "bs": "bos", + 'bs': 'bos', # Bulgarian - "bg": "bul", + 'bg': 'bul', # Catalan - "ca": "cat", + 'ca': 'cat', # Cebuano - "ceb": "ceb", + 'ceb': 'ceb', # Czech - "cs": "ces", + 'cs': 'ces', # Central Kurdish - "ckb": "ckb", - # Mandarin Chinese (Simplified) - "zh": "cmn", - # Mandarin Chinese (Traditional) - # (No separate 2-letter code) + 'ku': 'ckb', + # Mandarin Chinese + 'cmn': 'cmn_Hant', # Welsh - "cy": "cym", + 'cy': 'cym', # Danish - "da": "dan", + 'da': 'dan', # German - "de": "deu", + 'de': 'deu', # Greek - "el": "ell", + 'el': 'ell', # English - "en": "eng", + 'en': 'eng', # Estonian - "et": "est", + 'et': 'est', # Basque - "eu": "eus", + 'eu': 'eus', # Finnish - "fi": "fin", + 'fi': 'fin', # French - "fr": "fra", - # West Central Oromo - # (No 2-letter code) + 'fr': 'fra', # Irish - "ga": "gle", + 'ga': 'gle', + # West Central Oromo, + 'gaz': 'gaz', # Galician - "gl": "glg", + 'gl': 'glg', # Gujarati - "gu": "guj", + 'gu': 'guj', # Hebrew - "he": "heb", + 'he': 'heb', # Hindi - "hi": "hin", + 'hi': 'hin', # Croatian - "hr": "hrv", + 'hr': 'hrv', # Hungarian - "hu": "hun", + 'hu': 'hun', # Armenian - "hy": "hye", + 'hy': 'hye', # Igbo - "ig": "ibo", + 'ig': 'ibo', # Indonesian - "id": "ind", + 'id': 'ind', # Icelandic - "is": "isl", + 'is': 'isl', # Italian - "it": "ita", + 'it': 'ita', # Javanese - "jv": "jav", + 'jv': 'jav', # Japanese - "ja": "jpn", + 'ja': 'jpn', # Kannada - "kn": "kan", + 'kn': 'kan', # Georgian - "ka": "kat", + 'ka': 'kat', # Kazakh - "kk": "kaz", + 'kk': 'kaz', # Halh Mongolian - # (No 2-letter code) + 'khk': 'khk', # Khmer - "km": "khm", + 'km': 'khm', # Kyrgyz - "ky": "kir", + 'ky': 'kir', # Korean - "ko": "kor", + 'ko': 'kor', # Lao - "lo": "lao", + 'lo': 'lao', # Lithuanian - "lt": "lit", + 'lt': 'lit', # Ganda - "lg": "lug", + 'lg': 'lug', # Luo - "luo": "luo", + 'luo': 'luo', # Standard Latvian - "lv": "lvs", + 'lv': 'lvs', # Maithili - # (No 2-letter code) + 'mai': 'mai', # Malayalam - "ml": "mal", + 'ml': 'mal', # Marathi - "mr": "mar", + 'mr': 'mar', # Macedonian - "mk": "mkd", + 'mk': 'mkd', # Maltese - "mt": "mlt", + 'mt': 'mlt', # Meitei - # (No 2-letter code) + 'mni': 'mni', # Burmese - "my": "mya", + 'my': 'mya', # Dutch - "nl": "nld", + 'nl': 'nld', # Norwegian Nynorsk - "nn": "nno", + 'nn': 'nno', # Norwegian Bokmål - "nb": "nob", + 'nb': 'nob', # Nepali - "ne": "npi", + 'ne': 'npi', # Nyanja - "ny": "nya", + 'ny': 'nya', # Odia - "or": "ory", + 'or': 'ory', # Punjabi - "pa": "pan", + 'pa': 'pan', # Southern Pashto - # (No 2-letter code) + 'pbt': 'pbt', # Western Persian - "fa": "pes", + 'pes': 'pes', # Polish - "pl": "pol", + 'pl': 'pol', # Portuguese - "pt": "por", + 'pt': 'por', # Romanian - "ro": "ron", + 'ro': 'ron', # Russian - "ru": "rus", + 'ru': 'rus', # Slovak - "sk": "slk", + 'sk': 'slk', # Slovenian - "sl": "slv", + 'sl': 'slv', # Shona - "sn": "sna", + 'sn': 'sna', # Sindhi - "sd": "snd", + 'sd': 'snd', # Somali - "so": "som", + 'so': 'som', # Spanish - "es": "spa", + 'es': 'spa', # Serbian - "sr": "srp", + 'sr': 'srp', # Swedish - "sv": "swe", + 'sv': 'swe', # Swahili - "sw": "swh", + 'sw': 'swh', # Tamil - "ta": "tam", + 'ta': 'tam', # Telugu - "te": "tel", + 'te': 'tel', # Tajik - "tg": "tgk", + 'tg': 'tgk', # Tagalog - "tl": "tgl", + 'tl': 'tgl', # Thai - "th": "tha", + 'th': 'tha', # Turkish - "tr": "tur", + 'tr': 'tur', # Ukrainian - "uk": "ukr", + 'uk': 'ukr', # Urdu - "ur": "urd", + 'ur': 'urd', # Northern Uzbek - "uz": "uzn", + 'uz': 'uzn', # Vietnamese - "vi": "vie", + 'vi': 'vie', # Yoruba - "yo": "yor", + 'yo': 'yor', # Cantonese - # (No separate 2-letter code) + 'yue': 'yue', + # Standard Malay + 'ms': 'zsm', # Zulu - "zu": "zul", + 'zu': 'zul' } return seamless_lang_code.get(lang_code, "eng") @@ -408,10 +410,10 @@ def web(): result: dict @app.post("/translate", dependencies=[Depends(apikey_auth)]) - def translate( - text: str, - source_language: Annotated[str, Body(...)] = "en", - target_language: Annotated[str, Body(...)] = "fr", + async def translate( + text: str, + source_language: Annotated[str, Body(...)] = "en", + target_language: Annotated[str, Body(...)] = "fr", ) -> TranslateResponse: func = translatorstub.translate_text.spawn( text=text, diff --git a/server/reflector/processors/audio_transcript_banana.py b/server/reflector/processors/audio_transcript_banana.py index af8f647d..fe339eea 100644 --- a/server/reflector/processors/audio_transcript_banana.py +++ b/server/reflector/processors/audio_transcript_banana.py @@ -14,14 +14,15 @@ API will be a POST request to TRANSCRIPT_URL: """ +from pathlib import Path + +import httpx from reflector.processors.audio_transcript import AudioTranscriptProcessor from reflector.processors.audio_transcript_auto import AudioTranscriptAutoProcessor from reflector.processors.types import AudioFile, Transcript, Word from reflector.settings import settings from reflector.storage import Storage from reflector.utils.retry import retry -from pathlib import Path -import httpx class AudioTranscriptBananaProcessor(AudioTranscriptProcessor): diff --git a/server/reflector/processors/audio_transcript_whisper.py b/server/reflector/processors/audio_transcript_whisper.py index 972c636a..e3bd595b 100644 --- a/server/reflector/processors/audio_transcript_whisper.py +++ b/server/reflector/processors/audio_transcript_whisper.py @@ -1,7 +1,7 @@ +from faster_whisper import WhisperModel from reflector.processors.audio_transcript import AudioTranscriptProcessor from reflector.processors.audio_transcript_auto import AudioTranscriptAutoProcessor from reflector.processors.types import AudioFile, Transcript, Word -from faster_whisper import WhisperModel class AudioTranscriptWhisperProcessor(AudioTranscriptProcessor): diff --git a/server/reflector/processors/types.py b/server/reflector/processors/types.py index 4d0b3504..ce792b35 100644 --- a/server/reflector/processors/types.py +++ b/server/reflector/processors/types.py @@ -117,113 +117,204 @@ class FinalTitle(BaseModel): title: str +# https://github.com/facebookresearch/seamless_communication/tree/main/scripts/m4t/predict#supported-languages class TranslationLanguages(BaseModel): language_to_id_mapping: dict = { - "Afrikaans": "af", - "Albanian": "sq", - "Amharic": "am", - "Arabic": "ar", - "Armenian": "hy", - "Asturian": "ast", - "Azerbaijani": "az", - "Bashkir": "ba", - "Belarusian": "be", - "Bengali": "bn", - "Bosnian": "bs", - "Breton": "br", - "Bulgarian": "bg", - "Burmese": "my", - "Catalan; Valencian": "ca", - "Cebuano": "ceb", - "Central Khmer": "km", - "Chinese": "zh", - "Croatian": "hr", - "Czech": "cs", - "Danish": "da", - "Dutch; Flemish": "nl", - "English": "en", - "Estonian": "et", - "Finnish": "fi", - "French": "fr", - "Fulah": "ff", - "Gaelic; Scottish Gaelic": "gd", - "Galician": "gl", - "Ganda": "lg", - "Georgian": "ka", - "German": "de", - "Greeek": "el", - "Gujarati": "gu", - "Haitian; Haitian Creole": "ht", - "Hausa": "ha", - "Hebrew": "he", - "Hindi": "hi", - "Hungarian": "hu", - "Icelandic": "is", - "Igbo": "ig", - "Iloko": "ilo", - "Indonesian": "id", - "Irish": "ga", - "Italian": "it", - "Japanese": "ja", - "Javanese": "jv", - "Kannada": "kn", - "Kazakh": "kk", - "Korean": "ko", - "Lao": "lo", - "Latvian": "lv", - "Lingala": "ln", - "Lithuanian": "lt", - "Luxembourgish; Letzeburgesch": "lb", - "Macedonian": "mk", - "Malagasy": "mg", - "Malay": "ms", - "Malayalam": "ml", - "Marathi": "mr", - "Mongolian": "mn", - "Nepali": "ne", - "Northern Sotho": "ns", - "Norwegian": "no", - "Occitan": "oc", - "Oriya": "or", - "Panjabi; Punjabi": "pa", - "Persian": "fa", - "Polish": "pl", - "Portuguese": "pt", - "Pushto; Pashto": "ps", - "Romanian; Moldavian; Moldovan": "ro", - "Russian": "ru", - "Serbian": "sr", - "Sindhi": "sd", - "Sinhala; Sinhalese": "si", - "Slovak": "sk", - "Slovenian": "sl", - "Somali": "so", - "Spanish": "es", - "Sundanese": "su", - "Swahili": "sw", - "Swati": "ss", - "Swedish": "sv", - "Tagalog": "tl", - "Tamil": "ta", - "Thai": "th", - "Tswana": "tn", - "Turkish": "tr", - "Ukrainian": "uk", - "Urdu": "ur", - "Uzbek": "uz", - "Vietnamese": "vi", - "Welsh": "cy", - "Western Frisian": "fy", - "Wolof": "wo", - "Xhosa": "xh", - "Yiddish": "yi", - "Yoruba": "yo", - "Zulu": "zu", + # Afrikaans + "af": "afr", + # Amharic + "am": "amh", + # Modern Standard Arabic + "ar": "arb", + # Moroccan Arabic + "ary": "ary", + # Egyptian Arabic + "arz": "arz", + # Assamese + "as": "asm", + # North Azerbaijani + "az": "azj", + # Belarusian + "be": "bel", + # Bengali + "bn": "ben", + # Bosnian + "bs": "bos", + # Bulgarian + "bg": "bul", + # Catalan + "ca": "cat", + # Cebuano + "ceb": "ceb", + # Czech + "cs": "ces", + # Central Kurdish + "ku": "ckb", + # Mandarin Chinese + "cmn": "cmn_Hant", + # Welsh + "cy": "cym", + # Danish + "da": "dan", + # German + "de": "deu", + # Greek + "el": "ell", + # English + "en": "eng", + # Estonian + "et": "est", + # Basque + "eu": "eus", + # Finnish + "fi": "fin", + # French + "fr": "fra", + # Irish + "ga": "gle", + # West Central Oromo, + "gaz": "gaz", + # Galician + "gl": "glg", + # Gujarati + "gu": "guj", + # Hebrew + "he": "heb", + # Hindi + "hi": "hin", + # Croatian + "hr": "hrv", + # Hungarian + "hu": "hun", + # Armenian + "hy": "hye", + # Igbo + "ig": "ibo", + # Indonesian + "id": "ind", + # Icelandic + "is": "isl", + # Italian + "it": "ita", + # Javanese + "jv": "jav", + # Japanese + "ja": "jpn", + # Kannada + "kn": "kan", + # Georgian + "ka": "kat", + # Kazakh + "kk": "kaz", + # Halh Mongolian + "khk": "khk", + # Khmer + "km": "khm", + # Kyrgyz + "ky": "kir", + # Korean + "ko": "kor", + # Lao + "lo": "lao", + # Lithuanian + "lt": "lit", + # Ganda + "lg": "lug", + # Luo + "luo": "luo", + # Standard Latvian + "lv": "lvs", + # Maithili + "mai": "mai", + # Malayalam + "ml": "mal", + # Marathi + "mr": "mar", + # Macedonian + "mk": "mkd", + # Maltese + "mt": "mlt", + # Meitei + "mni": "mni", + # Burmese + "my": "mya", + # Dutch + "nl": "nld", + # Norwegian Nynorsk + "nn": "nno", + # Norwegian Bokmål + "nb": "nob", + # Nepali + "ne": "npi", + # Nyanja + "ny": "nya", + # Odia + "or": "ory", + # Punjabi + "pa": "pan", + # Southern Pashto + "pbt": "pbt", + # Western Persian + "pes": "pes", + # Polish + "pl": "pol", + # Portuguese + "pt": "por", + # Romanian + "ro": "ron", + # Russian + "ru": "rus", + # Slovak + "sk": "slk", + # Slovenian + "sl": "slv", + # Shona + "sn": "sna", + # Sindhi + "sd": "snd", + # Somali + "so": "som", + # Spanish + "es": "spa", + # Serbian + "sr": "srp", + # Swedish + "sv": "swe", + # Swahili + "sw": "swh", + # Tamil + "ta": "tam", + # Telugu + "te": "tel", + # Tajik + "tg": "tgk", + # Tagalog + "tl": "tgl", + # Thai + "th": "tha", + # Turkish + "tr": "tur", + # Ukrainian + "uk": "ukr", + # Urdu + "ur": "urd", + # Northern Uzbek + "uz": "uzn", + # Vietnamese + "vi": "vie", + # Yoruba + "yo": "yor", + # Cantonese + "yue": "yue", + # Standard Malay + "ms": "zsm", + # Zulu + "zu": "zul", } @property def supported_languages(self): - return self.language_to_id_mapping.values() + return self.language_to_id_mapping.keys() def is_supported(self, lang_id: str) -> bool: if lang_id in self.supported_languages: diff --git a/server/reflector/tools/process.py b/server/reflector/tools/process.py index 37f44096..d619040d 100644 --- a/server/reflector/tools/process.py +++ b/server/reflector/tools/process.py @@ -1,7 +1,6 @@ import asyncio import av - from reflector.logger import logger from reflector.processors import ( AudioChunkerProcessor, diff --git a/www/app/supportedLanguages.ts b/www/app/supportedLanguages.ts new file mode 100644 index 00000000..4285e041 --- /dev/null +++ b/www/app/supportedLanguages.ts @@ -0,0 +1,495 @@ +import Script from "next/script"; + +// type Script = 'Latn' | 'Ethi' | 'Arab' | 'Beng' | 'Cyrl' | 'Taml' | 'Hant' | 'Hans' | 'Grek' | 'Gujr' | 'Hebr'| 'Deva'| 'Armn' | 'Jpan' | 'Knda' | 'Geor'; +type LanguageOption = { + value: string | undefined; + name: string; + script?: string; +}; + +const supportedLanguages: LanguageOption[] = [ + { + value: "af", + name: "Afrikaans", + script: "Latn", + }, + { + value: "am", + name: "Amharic", + script: "Ethi", + }, + { + value: "ar", + name: "Modern Standard Arabic", + script: "Arab", + }, + { + value: "ary", + name: "Moroccan Arabic", + script: "Arab", + }, + { + value: "arz", + name: "Egyptian Arabic", + script: "Arab", + }, + { + value: "as", + name: "Assamese", + script: "Beng", + }, + { + value: "az", + name: "North Azerbaijani", + script: "Latn", + }, + { + value: "be", + name: "Belarusian", + script: "Cyrl", + }, + { + value: "bn", + name: "Bengali", + script: "Beng", + }, + { + value: "bs", + name: "Bosnian", + script: "Latn", + }, + { + value: "bg", + name: "Bulgarian", + script: "Cyrl", + }, + { + value: "ca", + name: "Catalan", + script: "Latn", + }, + { + value: "ceb", + name: "Cebuano", + script: "Latn", + }, + { + value: "cs", + name: "Czech", + script: "Latn", + }, + { + value: "ku", + name: "Central Kurdish", + script: "Arab", + }, + { + value: "cmn", + name: "Mandarin Chinese", + script: "Hans", + }, + { + value: "cy", + name: "Welsh", + script: "Latn", + }, + { + value: "da", + name: "Danish", + script: "Latn", + }, + { + value: "de", + name: "German", + script: "Latn", + }, + { + value: "el", + name: "Greek", + script: "Grek", + }, + { + value: "en", + name: "English", + script: "Latn", + }, + { + value: "et", + name: "Estonian", + script: "Latn", + }, + { + value: "eu", + name: "Basque", + script: "Latn", + }, + { + value: "fi", + name: "Finnish", + script: "Latn", + }, + { + value: "fr", + name: "French", + script: "Latn", + }, + { + value: "gaz", + name: "West Central Oromo", + script: "Latn", + }, + { + value: "ga", + name: "Irish", + script: "Latn", + }, + { + value: "gl", + name: "Galician", + script: "Latn", + }, + { + value: "gu", + name: "Gujarati", + script: "Gujr", + }, + { + value: "he", + name: "Hebrew", + script: "Hebr", + }, + { + value: "hi", + name: "Hindi", + script: "Deva", + }, + { + value: "hr", + name: "Croatian", + script: "Latn", + }, + { + value: "hu", + name: "Hungarian", + script: "Latn", + }, + { + value: "hy", + name: "Armenian", + script: "Armn", + }, + { + value: "ig", + name: "Igbo", + script: "Latn", + }, + { + value: "id", + name: "Indonesian", + script: "Latn", + }, + { + value: "is", + name: "Icelandic", + script: "Latn", + }, + { + value: "it", + name: "Italian", + script: "Latn", + }, + { + value: "jv", + name: "Javanese", + script: "Latn", + }, + { + value: "ja", + name: "Japanese", + script: "Jpan", + }, + { + value: "kn", + name: "Kannada", + script: "Knda", + }, + { + value: "ka", + name: "Georgian", + script: "Geor", + }, + { + value: "kk", + name: "Kazakh", + script: "Cyrl", + }, + { + value: "khk", + name: "Halh Mongolian", + script: "Cyrl", + }, + { + value: "km", + name: "Khmer", + script: "Khmr", + }, + { + value: "ky", + name: "Kyrgyz", + script: "Cyrl", + }, + { + value: "ko", + name: "Korean", + script: "Kore", + }, + { + value: "lo", + name: "Lao", + script: "Laoo", + }, + { + value: "lt", + name: "Lithuanian", + script: "Latn", + }, + { + value: "lg", + name: "Ganda", + script: "Latn", + }, + { + value: "luo", + name: "Luo", + script: "Latn", + }, + { + value: "lv", + name: "Standard Latvian", + script: "Latn", + }, + { + value: "mai", + name: "Maithili", + script: "Deva", + }, + { + value: "ml", + name: "Malayalam", + script: "Mlym", + }, + { + value: "mr", + name: "Marathi", + script: "Deva", + }, + { + value: "mk", + name: "Macedonian", + script: "Cyrl", + }, + { + value: "mt", + name: "Maltese", + script: "Latn", + }, + { + value: "mni", + name: "Meitei", + script: "Beng", + }, + { + value: "my", + name: "Burmese", + script: "Mymr", + }, + { + value: "nl", + name: "Dutch", + script: "Latn", + }, + { + value: "nn", + name: "Norwegian Nynorsk", + script: "Latn", + }, + { + value: "nb", + name: "Norwegian Bokmål", + script: "Latn", + }, + { + value: "ne", + name: "Nepali", + script: "Deva", + }, + { + value: "ny", + name: "Nyanja", + script: "Latn", + }, + { + value: "or", + name: "Odia", + script: "Orya", + }, + { + value: "pa", + name: "Punjabi", + script: "Guru", + }, + { + value: "pbt", + name: "Southern Pashto", + script: "Arab", + }, + { + value: "pes", + name: "Western Persian", + script: "Arab", + }, + { + value: "pl", + name: "Polish", + script: "Latn", + }, + { + value: "pt", + name: "Portuguese", + script: "Latn", + }, + { + value: "ro", + name: "Romanian", + script: "Latn", + }, + { + value: "ru", + name: "Russian", + script: "Cyrl", + }, + { + value: "sk", + name: "Slovak", + script: "Latn", + }, + { + value: "sl", + name: "Slovenian", + script: "Latn", + }, + { + value: "sn", + name: "Shona", + script: "Latn", + }, + { + value: "sd", + name: "Sindhi", + script: "Arab", + }, + { + value: "so", + name: "Somali", + script: "Latn", + }, + { + value: "es", + name: "Spanish", + script: "Latn", + }, + { + value: "sr", + name: "Serbian", + script: "Cyrl", + }, + { + value: "sv", + name: "Swedish", + script: "Latn", + }, + { + value: "sw", + name: "Swahili", + script: "Latn", + }, + { + value: "ta", + name: "Tamil", + script: "Taml", + }, + { + value: "te", + name: "Telugu", + script: "Telu", + }, + { + value: "tg", + name: "Tajik", + script: "Cyrl", + }, + { + value: "tl", + name: "Tagalog", + script: "Latn", + }, + { + value: "th", + name: "Thai", + script: "Thai", + }, + { + value: "tr", + name: "Turkish", + script: "Latn", + }, + { + value: "uk", + name: "Ukrainian", + script: "Cyrl", + }, + { + value: "ur", + name: "Urdu", + script: "Arab", + }, + { + value: "uz", + name: "Northern Uzbek", + script: "Latn", + }, + { + value: "vi", + name: "Vietnamese", + script: "Latn", + }, + { + value: "yo", + name: "Yoruba", + script: "Latn", + }, + { + value: "yue", + name: "Cantonese", + script: "Hant", + }, + { + value: "ms", + name: "Standard Malay", + script: "Latn", + }, + { + value: "zu", + name: "Zulu", + script: "Latn", + }, +]; + +const supportedLatinLanguages = supportedLanguages.filter( + (lan) => lan.script == "Latn", +); +supportedLatinLanguages.push({ value: undefined, name: "None" }); + +export { supportedLatinLanguages }; + +export default supportedLanguages; diff --git a/www/app/transcripts/[transcriptId]/page.tsx b/www/app/transcripts/[transcriptId]/page.tsx index d1d985c7..c6dbb501 100644 --- a/www/app/transcripts/[transcriptId]/page.tsx +++ b/www/app/transcripts/[transcriptId]/page.tsx @@ -22,7 +22,7 @@ type TranscriptDetails = { export default function TranscriptDetails(details: TranscriptDetails) { const api = getApi(); - const transcript = useTranscript(null, api, details.params.transcriptId); + const transcript = useTranscript(details.params.transcriptId); const topics = useTopics(api, details.params.transcriptId); const waveform = useWaveform(api, details.params.transcriptId); const useActiveTopic = useState(null); diff --git a/www/app/transcripts/[transcriptId]/record/page.tsx b/www/app/transcripts/[transcriptId]/record/page.tsx new file mode 100644 index 00000000..2e212c2e --- /dev/null +++ b/www/app/transcripts/[transcriptId]/record/page.tsx @@ -0,0 +1,142 @@ +"use client"; +import React, { useEffect, useState } from "react"; +import Recorder from "../../recorder"; +import { TopicList } from "../../topicList"; +import useWebRTC from "../../useWebRTC"; +import useTranscript from "../../useTranscript"; +import { useWebSockets } from "../../useWebSockets"; +import useAudioDevice from "../../useAudioDevice"; +import "../../../styles/button.css"; +import { Topic } from "../../webSocketTypes"; +import getApi from "../../../lib/getApi"; +import LiveTrancription from "../../liveTranscription"; +import DisconnectedIndicator from "../../disconnectedIndicator"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faGear } from "@fortawesome/free-solid-svg-icons"; +import { lockWakeState, releaseWakeState } from "../../../lib/wakeLock"; + +type TranscriptDetails = { + params: { + transcriptId: string; + }; +}; + +const TranscriptRecord = (details: TranscriptDetails) => { + const [stream, setStream] = useState(null); + const [disconnected, setDisconnected] = useState(false); + const useActiveTopic = useState(null); + + useEffect(() => { + if (process.env.NEXT_PUBLIC_ENV === "development") { + document.onkeyup = (e) => { + if (e.key === "d") { + setDisconnected((prev) => !prev); + } + }; + } + }, []); + + const transcript = useTranscript(details.params.transcriptId); + const api = getApi(); + const webRTC = useWebRTC(stream, details.params.transcriptId, api); + const webSockets = useWebSockets(details.params.transcriptId); + + const { audioDevices, getAudioStream } = useAudioDevice(); + + const [hasRecorded, setHasRecorded] = useState(false); + const [transcriptStarted, setTranscriptStarted] = useState(false); + + useEffect(() => { + if (!transcriptStarted && webSockets.transcriptText.length !== 0) + setTranscriptStarted(true); + }, [webSockets.transcriptText]); + + useEffect(() => { + if (transcript?.response?.longSummary) { + const newUrl = `/transcripts/${transcript.response.id}`; + // Shallow redirection does not work on NextJS 13 + // https://github.com/vercel/next.js/discussions/48110 + // https://github.com/vercel/next.js/discussions/49540 + // router.push(newUrl, undefined, { shallow: true }); + history.replaceState({}, "", newUrl); + } + }); + + useEffect(() => { + lockWakeState(); + return () => { + releaseWakeState(); + }; + }, []); + + return ( + <> + { + setStream(null); + setHasRecorded(true); + webRTC?.send(JSON.stringify({ cmd: "STOP" })); + }} + topics={webSockets.topics} + getAudioStream={getAudioStream} + useActiveTopic={useActiveTopic} + isPastMeeting={false} + audioDevices={audioDevices} + /> + +
+ + +
+ {!hasRecorded ? ( + <> + {transcriptStarted && ( +

Transcription

+ )} +
+
+ {!transcriptStarted ? ( +
+ The conversation transcript will appear here shortly after + you start recording. +
+ ) : ( + + )} +
+
+ + ) : ( +
+
+ +
+

+ We are generating the final summary for you. This may take a + couple of minutes. Please do not navigate away from the page + during this time. +

+
+ )} +
+
+ + {disconnected && } + + ); +}; + +export default TranscriptRecord; diff --git a/www/app/transcripts/createTranscript.ts b/www/app/transcripts/createTranscript.ts new file mode 100644 index 00000000..32024cb4 --- /dev/null +++ b/www/app/transcripts/createTranscript.ts @@ -0,0 +1,54 @@ +import { useEffect, useState } from "react"; +import { DefaultApi, V1TranscriptsCreateRequest } from "../api/apis/DefaultApi"; +import { GetTranscript } from "../api"; +import { useError } from "../(errors)/errorContext"; +import getApi from "../lib/getApi"; + +type CreateTranscript = { + response: GetTranscript | null; + loading: boolean; + error: Error | null; + create: (params: V1TranscriptsCreateRequest["createTranscript"]) => void; +}; + +const useCreateTranscript = (): CreateTranscript => { + const [response, setResponse] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setErrorState] = useState(null); + const { setError } = useError(); + const api = getApi(); + + const create = (params: V1TranscriptsCreateRequest["createTranscript"]) => { + if (loading) return; + + setLoading(true); + const requestParameters: V1TranscriptsCreateRequest = { + createTranscript: { + name: params.name || "Weekly All-Hands", // Default + targetLanguage: params.targetLanguage || "en", // Default + }, + }; + + console.debug( + "POST - /v1/transcripts/ - Requesting new transcription creation", + requestParameters, + ); + + api + .v1TranscriptsCreate(requestParameters) + .then((result) => { + setResponse(result); + setLoading(false); + console.debug("New transcript created:", result); + }) + .catch((err) => { + setError(err); + setErrorState(err); + setLoading(false); + }); + }; + + return { response, loading, error, create }; +}; + +export default useCreateTranscript; diff --git a/www/app/transcripts/liveTranscription.tsx b/www/app/transcripts/liveTranscription.tsx index 2c913a91..6e40e7a3 100644 --- a/www/app/transcripts/liveTranscription.tsx +++ b/www/app/transcripts/liveTranscription.tsx @@ -1,14 +1,19 @@ type LiveTranscriptionProps = { text: string; + translateText: string; }; export default function LiveTrancription(props: LiveTranscriptionProps) { return (

- {/* Nous allons prendre quelques appels téléphoniques et répondre à quelques questions */} - {props.text} + {props.translateText ? props.translateText : props.text}

+ {props.translateText && ( +

+ {props.text} +

+ )}
); } diff --git a/www/app/transcripts/new/page.tsx b/www/app/transcripts/new/page.tsx index b4e294d6..f2b15120 100644 --- a/www/app/transcripts/new/page.tsx +++ b/www/app/transcripts/new/page.tsx @@ -1,197 +1,112 @@ "use client"; import React, { useEffect, useState } from "react"; -import Recorder from "../recorder"; -import { TopicList } from "../topicList"; -import useWebRTC from "../useWebRTC"; -import useTranscript from "../useTranscript"; -import { useWebSockets } from "../useWebSockets"; import useAudioDevice from "../useAudioDevice"; import "../../styles/button.css"; -import { Topic } from "../webSocketTypes"; import getApi from "../../lib/getApi"; -import LiveTrancription from "../liveTranscription"; -import DisconnectedIndicator from "../disconnectedIndicator"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faGear } from "@fortawesome/free-solid-svg-icons"; import About from "../../(aboutAndPrivacy)/about"; import Privacy from "../../(aboutAndPrivacy)/privacy"; -import { lockWakeState, releaseWakeState } from "../../lib/wakeLock"; import { useRouter } from "next/navigation"; +import useCreateTranscript from "../createTranscript"; +import SelectSearch from "react-select-search"; +import { supportedLatinLanguages } from "../../supportedLanguages"; +import "react-select-search/style.css"; const TranscriptCreate = () => { - const [stream, setStream] = useState(null); - const [disconnected, setDisconnected] = useState(false); - const useActiveTopic = useState(null); - - useEffect(() => { - if (process.env.NEXT_PUBLIC_ENV === "development") { - document.onkeyup = (e) => { - if (e.key === "d") { - setDisconnected((prev) => !prev); - } - }; - } - }, []); - - const api = getApi(); - const transcript = useTranscript(stream, api); - const webRTC = useWebRTC(stream, transcript?.response?.id, api); - const webSockets = useWebSockets(transcript?.response?.id); + // const transcript = useTranscript(stream, api); const router = useRouter(); - const { - loading, - permissionOk, - permissionDenied, - audioDevices, - requestPermission, - getAudioStream, - } = useAudioDevice(); + const api = getApi(); - const [hasRecorded, setHasRecorded] = useState(false); - const [transcriptStarted, setTranscriptStarted] = useState(false); + const [name, setName] = useState(); + const nameChange = (event: React.ChangeEvent) => { + setName(event.target.value); + }; + const [targetLanguage, setTargetLanguage] = useState(); + const onLanguageChange = (newval) => { + typeof newval === "string" && setTargetLanguage(newval); + }; + + const createTranscript = useCreateTranscript(); + + const send = () => { + if (createTranscript.loading || permissionDenied) return; + createTranscript.create({ name, targetLanguage }); + }; useEffect(() => { - if (!transcriptStarted && webSockets.transcriptText.length !== 0) - setTranscriptStarted(true); - }, [webSockets.transcriptText]); + createTranscript.response && + router.push(`/transcripts/${createTranscript.response.id}/record`); + }, [createTranscript.response]); - useEffect(() => { - if (transcript?.response?.id) { - const newUrl = `/transcripts/${transcript.response.id}`; - // Shallow redirection does not work on NextJS 13 - // https://github.com/vercel/next.js/discussions/48110 - // https://github.com/vercel/next.js/discussions/49540 - // router.push(newUrl, undefined, { shallow: true }); - history.replaceState({}, "", newUrl); - } - }); - - useEffect(() => { - lockWakeState(); - return () => { - releaseWakeState(); - }; - }, []); + const { loading, permissionOk, permissionDenied, requestPermission } = + useAudioDevice(); return ( <> - {permissionOk ? ( - <> - { - webRTC?.send(JSON.stringify({ cmd: "STOP" })); - setStream(null); - setHasRecorded(true); - }} - topics={webSockets.topics} - getAudioStream={getAudioStream} - useActiveTopic={useActiveTopic} - isPastMeeting={false} - audioDevices={audioDevices} - /> +
+
+
+
+

+ Welcome to reflector.media +

+

+ Reflector is a transcription and summarization pipeline that + transforms audio into knowledge. The output is meeting minutes and + topic summaries enabling topic-specific analyses stored in your + systems of record. This is accomplished on your infrastructure – + without 3rd parties – keeping your data private, secure, and + organized. +

+ +
+
+
+

Try Reflector

+ -
- +

Do you need live translation ?

+ + -
- {!hasRecorded ? ( - <> - {transcriptStarted && ( -

Transcription

- )} -
-
- {!transcriptStarted ? ( -
- The conversation transcript will appear here shortly - after you start recording. -
- ) : ( - - )} -
-
- - ) : ( -
-
- -
-

- We are generating the final summary for you. This may take a - couple of minutes. Please do not navigate away from the page - during this time. -

-
- )} -
-
- - {disconnected && } - - ) : ( - <> -
-
-
-
-
-

- Welcome to reflector.media -

-

- Reflector is a transcription and summarization pipeline that - transforms audio into knowledge. The output is meeting - minutes and topic summaries enabling topic-specific analyses - stored in your systems of record. This is accomplished on - your infrastructure – without 3rd parties – keeping your - data private, secure, and organized. -

- -

- Audio Permissions -

- {loading ? ( -

Checking permission...

- ) : ( - <> -

- In order to use Reflector, we kindly request permission - to access your microphone during meetings and events. -
- -
- {permissionDenied - ? "Permission to use your microphone was denied, please change the permission setting in your browser and refresh this page." - : "Please grant permission to continue."} -

- - - )} -
-
-
-
- - )} + {loading ? ( +

Checking permission...

+ ) : permissionOk ? ( + <> Microphone permission granted + ) : ( + <> +

+ In order to use Reflector, we kindly request permission to + access your microphone during meetings and events. +
+ +
+ {permissionDenied && + "Permission to use your microphone was denied, please change the permission setting in your browser and refresh this page."} +

+ + + )} + +
+
); }; diff --git a/www/app/transcripts/useTranscript.ts b/www/app/transcripts/useTranscript.ts index 7dbc969a..eb39b202 100644 --- a/www/app/transcripts/useTranscript.ts +++ b/www/app/transcripts/useTranscript.ts @@ -1,11 +1,11 @@ import { useEffect, useState } from "react"; import { - DefaultApi, V1TranscriptGetRequest, V1TranscriptsCreateRequest, } from "../api/apis/DefaultApi"; import { GetTranscript } from "../api"; import { useError } from "../(errors)/errorContext"; +import getApi from "../lib/getApi"; type Transcript = { response: GetTranscript | null; @@ -13,23 +13,12 @@ type Transcript = { error: Error | null; }; -const useTranscript = ( - stream: MediaStream | null, - api: DefaultApi, - id: string | null = null, -): Transcript => { +const useTranscript = (id: string | null): Transcript => { const [response, setResponse] = useState(null); const [loading, setLoading] = useState(false); const [error, setErrorState] = useState(null); const { setError } = useError(); - - const getOrCreateTranscript = (id: string | null) => { - if (id) { - getTranscript(id); - } else if (stream) { - createTranscript(); - } - }; + const api = getApi(); const getTranscript = (id: string | null) => { if (!id) throw new Error("Transcript ID is required to get transcript"); @@ -43,34 +32,7 @@ const useTranscript = ( .then((result) => { setResponse(result); setLoading(false); - console.debug("New transcript created:", result); - }) - .catch((err) => { - setError(err); - setErrorState(err); - }); - }; - - const createTranscript = () => { - setLoading(true); - const requestParameters: V1TranscriptsCreateRequest = { - createTranscript: { - name: "Weekly All-Hands", // Hardcoded for now - targetLanguage: "en", // Hardcoded for now - }, - }; - - console.debug( - "POST - /v1/transcripts/ - Requesting new transcription creation", - requestParameters, - ); - - api - .v1TranscriptsCreate(requestParameters) - .then((result) => { - setResponse(result); - setLoading(false); - console.debug("New transcript created:", result); + console.debug("Transcript Loaded:", result); }) .catch((err) => { setError(err); @@ -79,8 +41,8 @@ const useTranscript = ( }; useEffect(() => { - getOrCreateTranscript(id); - }, [id, stream]); + getTranscript(id); + }, [id]); return { response, loading, error }; }; diff --git a/www/app/transcripts/useWebSockets.ts b/www/app/transcripts/useWebSockets.ts index cb31c4bf..a9e0cf57 100644 --- a/www/app/transcripts/useWebSockets.ts +++ b/www/app/transcripts/useWebSockets.ts @@ -5,6 +5,7 @@ import { useRouter } from "next/navigation"; type UseWebSockets = { transcriptText: string; + translateText: string; topics: Topic[]; finalSummary: FinalSummary; status: Status; @@ -12,7 +13,9 @@ type UseWebSockets = { export const useWebSockets = (transcriptId: string | null): UseWebSockets => { const [transcriptText, setTranscriptText] = useState(""); + const [translateText, setTranslateText] = useState(""); const [textQueue, setTextQueue] = useState([]); + const [translationQueue, setTranslationQueue] = useState([]); const [isProcessing, setIsProcessing] = useState(false); const [topics, setTopics] = useState([]); const [finalSummary, setFinalSummary] = useState({ @@ -30,6 +33,8 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { setIsProcessing(true); const text = textQueue[0]; setTranscriptText(text); + setTranslateText(translationQueue[0]); + console.log("displaying " + translateText); const WPM_READING = 200 + textQueue.length * 10; // words per minute to read const wordCount = text.split(/\s+/).length; @@ -38,6 +43,7 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { setTimeout(() => { setIsProcessing(false); setTextQueue((prevQueue) => prevQueue.slice(1)); + setTranslationQueue((prevQueue) => prevQueue.slice(1)); }, delay); }, [textQueue, isProcessing]); @@ -158,11 +164,13 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { switch (message.event) { case "TRANSCRIPT": const newText = (message.data.text ?? "").trim(); + const newTranslation = (message.data.translation ?? "").trim(); if (!newText) break; console.debug("TRANSCRIPT event:", newText); setTextQueue((prevQueue) => [...prevQueue, newText]); + setTranslationQueue((prevQueue) => [...prevQueue, newTranslation]); break; case "TOPIC": @@ -233,5 +241,5 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => { }; }, [transcriptId]); - return { transcriptText, topics, finalSummary, status }; + return { transcriptText, translateText, topics, finalSummary, status }; }; diff --git a/www/package.json b/www/package.json index 407f5512..91de6ef7 100644 --- a/www/package.json +++ b/www/package.json @@ -27,6 +27,7 @@ "react-dropdown": "^1.11.0", "react-markdown": "^9.0.0", "react-qr-code": "^2.0.12", + "react-select-search": "^4.1.7", "sass": "^1.63.6", "simple-peer": "^9.11.1", "superagent": "^8.0.9", diff --git a/www/yarn.lock b/www/yarn.lock index 5902f7a7..d3d39205 100644 --- a/www/yarn.lock +++ b/www/yarn.lock @@ -2088,6 +2088,11 @@ react-qr-code@^2.0.12: prop-types "^15.8.1" qr.js "0.0.0" +react-select-search@^4.1.7: + version "4.1.7" + resolved "https://registry.yarnpkg.com/react-select-search/-/react-select-search-4.1.7.tgz#5662729b9052282bde52e1352006d495d9c5ed6e" + integrity sha512-pU7ONAdK+bmz2tbhBWYQv9m5mnXOn8yImuiy+5UhimIG80d5iKv3nSYJIjJWjDbdrrdoXiCRwQm8xbA8llTjmQ== + react@^18.2.0: version "18.2.0" resolved "https://registry.npmjs.org/react/-/react-18.2.0.tgz"