selfhostyourtech/reflector
2
0
Fork 0
mirror of https://github.com/Monadical-SAS/reflector.git synced 2025-12-21 20:59:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: daily.co support as alternative to whereby (#691)

Browse Source 
* llm instructions

* vibe dailyco

* vibe dailyco

* doc update (vibe)

* dont show recording ui on call

* stub processor (vibe)

* stub processor (vibe) self-review

* stub processor (vibe) self-review

* chore(main): release 0.14.0 (#670)

* Add multitrack pipeline

* Mixdown audio tracks

* Mixdown with pyav filter graph

* Trigger multitrack processing for daily recordings

* apply platform from envs in priority: non-dry

* Use explicit track keys for processing

* Align tracks of a multitrack recording

* Generate waveforms for the mixed audio

* Emit multriack pipeline events

* Fix multitrack pipeline track alignment

* dailico docs

* Enable multitrack reprocessing

* modal temp files uniform names, cleanup. remove llm temporary docs

* docs cleanup

* dont proceed with raw recordings if any of the downloads fail

* dry transcription pipelines

* remove is_miltitrack

* comments

* explicit dailyco room name

* docs

* remove stub data/method

* frontend daily/whereby code self-review (no-mistake)

* frontend daily/whereby code self-review (no-mistakes)

* frontend daily/whereby code self-review (no-mistakes)

* consent cleanup for multitrack (no-mistakes)

* llm fun

* remove extra comments

* fix tests

* merge migrations

* Store participant names

* Get participants by meeting session id

* pop back main branch migration

* s3 paddington (no-mistakes)

* comment

* pr comments

* pr comments

* pr comments

* platform / meeting cleanup

* Use participant names in summary generation

* platform assignment to meeting at controller level

* pr comment

* room playform properly default none

* room playform properly default none

* restore migration lost

* streaming WIP

* extract storage / use common storage / proper env vars for storage

* fix mocks tests

* remove fall back

* streaming for multifile

* cenrtal storage abstraction (no-mistakes)

* remove dead code / vars

* Set participant user id for authenticated users

* whereby recording name parsing fix

* whereby recording name parsing fix

* more file stream

* storage dry + tests

* remove homemade boto3 streaming and use proper boto

* update migration guide

* webhook creation script - print uuid

---------

Co-authored-by: Igor Loskutov <igor.loskutoff@gmail.com>
Co-authored-by: Mathieu Virbel <mat@meltingrocks.com>
Co-authored-by: Sergey Mankovsky <sergey@monadical.com>
This commit is contained in:
Igor Monadical
2025-11-12 21:21:16 -05:00
committed by GitHub
parent 372202b0e1
commit 1473fd82dc
71 changed files with 4985 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files

330
server/tests/test_consent_multitrack.py Normal file
View File

@@ -0,0 +1,330 @@
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from reflector.db.meetings import (
MeetingConsent,
meeting_consent_controller,
meetings_controller,
)
from reflector.db.recordings import Recording, recordings_controller
from reflector.db.rooms import rooms_controller
from reflector.db.transcripts import SourceKind, transcripts_controller
from reflector.pipelines.main_live_pipeline import cleanup_consent
@pytest.mark.asyncio
async def test_consent_cleanup_deletes_multitrack_files():
room = await rooms_controller.add(
name="Test Room",
user_id="test-user",
zulip_auto_post=False,
zulip_stream="",
zulip_topic="",
is_locked=False,
room_mode="normal",
recording_type="cloud",
recording_trigger="automatic",
is_shared=False,
platform="daily",
)
# Create meeting
meeting = await meetings_controller.create(
id="test-multitrack-meeting",
room_name="test-room-20250101120000",
room_url="https://test.daily.co/test-room",
host_room_url="https://test.daily.co/test-room",
start_date=datetime.now(timezone.utc),
end_date=datetime.now(timezone.utc),
room=room,
)
track_keys = [
"recordings/test-room-20250101120000/track-0.webm",
"recordings/test-room-20250101120000/track-1.webm",
"recordings/test-room-20250101120000/track-2.webm",
]
recording = await recordings_controller.create(
Recording(
bucket_name="test-bucket",
object_key="recordings/test-room-20250101120000", # Folder path
recorded_at=datetime.now(timezone.utc),
meeting_id=meeting.id,
track_keys=track_keys,
)
)
# Create transcript
transcript = await transcripts_controller.add(
name="Test Multitrack Transcript",
source_kind=SourceKind.ROOM,
recording_id=recording.id,
meeting_id=meeting.id,
)
# Add consent denial
await meeting_consent_controller.upsert(
MeetingConsent(
meeting_id=meeting.id,
user_id="test-user",
consent_given=False,
consent_timestamp=datetime.now(timezone.utc),
)
)
# Mock get_transcripts_storage (master credentials with bucket override)
with patch(
"reflector.pipelines.main_live_pipeline.get_transcripts_storage"
) as mock_get_transcripts_storage:
mock_master_storage = MagicMock()
mock_master_storage.delete_file = AsyncMock()
mock_get_transcripts_storage.return_value = mock_master_storage
await cleanup_consent(transcript_id=transcript.id)
# Verify master storage was used with bucket override for all track keys
assert mock_master_storage.delete_file.call_count == 3
deleted_keys = []
for call_args in mock_master_storage.delete_file.call_args_list:
key = call_args[0][0]
bucket_kwarg = call_args[1].get("bucket")
deleted_keys.append(key)
assert bucket_kwarg == "test-bucket" # Verify bucket override!
assert set(deleted_keys) == set(track_keys)
updated_transcript = await transcripts_controller.get_by_id(transcript.id)
assert updated_transcript.audio_deleted is True
@pytest.mark.asyncio
async def test_consent_cleanup_handles_missing_track_keys():
room = await rooms_controller.add(
name="Test Room 2",
user_id="test-user",
zulip_auto_post=False,
zulip_stream="",
zulip_topic="",
is_locked=False,
room_mode="normal",
recording_type="cloud",
recording_trigger="automatic",
is_shared=False,
platform="daily",
)
# Create meeting
meeting = await meetings_controller.create(
id="test-multitrack-meeting-2",
room_name="test-room-20250101120001",
room_url="https://test.daily.co/test-room-2",
host_room_url="https://test.daily.co/test-room-2",
start_date=datetime.now(timezone.utc),
end_date=datetime.now(timezone.utc),
room=room,
)
recording = await recordings_controller.create(
Recording(
bucket_name="test-bucket",
object_key="recordings/old-style-recording.mp4",
recorded_at=datetime.now(timezone.utc),
meeting_id=meeting.id,
track_keys=None,
)
)
transcript = await transcripts_controller.add(
name="Test Old-Style Transcript",
source_kind=SourceKind.ROOM,
recording_id=recording.id,
meeting_id=meeting.id,
)
# Add consent denial
await meeting_consent_controller.upsert(
MeetingConsent(
meeting_id=meeting.id,
user_id="test-user-2",
consent_given=False,
consent_timestamp=datetime.now(timezone.utc),
)
)
# Mock get_transcripts_storage (master credentials with bucket override)
with patch(
"reflector.pipelines.main_live_pipeline.get_transcripts_storage"
) as mock_get_transcripts_storage:
mock_master_storage = MagicMock()
mock_master_storage.delete_file = AsyncMock()
mock_get_transcripts_storage.return_value = mock_master_storage
await cleanup_consent(transcript_id=transcript.id)
# Verify master storage was used with bucket override
assert mock_master_storage.delete_file.call_count == 1
call_args = mock_master_storage.delete_file.call_args
assert call_args[0][0] == recording.object_key
assert call_args[1].get("bucket") == "test-bucket" # Verify bucket override!
@pytest.mark.asyncio
async def test_consent_cleanup_empty_track_keys_falls_back():
room = await rooms_controller.add(
name="Test Room 3",
user_id="test-user",
zulip_auto_post=False,
zulip_stream="",
zulip_topic="",
is_locked=False,
room_mode="normal",
recording_type="cloud",
recording_trigger="automatic",
is_shared=False,
platform="daily",
)
# Create meeting
meeting = await meetings_controller.create(
id="test-multitrack-meeting-3",
room_name="test-room-20250101120002",
room_url="https://test.daily.co/test-room-3",
host_room_url="https://test.daily.co/test-room-3",
start_date=datetime.now(timezone.utc),
end_date=datetime.now(timezone.utc),
room=room,
)
recording = await recordings_controller.create(
Recording(
bucket_name="test-bucket",
object_key="recordings/fallback-recording.mp4",
recorded_at=datetime.now(timezone.utc),
meeting_id=meeting.id,
track_keys=[],
)
)
transcript = await transcripts_controller.add(
name="Test Empty Track Keys Transcript",
source_kind=SourceKind.ROOM,
recording_id=recording.id,
meeting_id=meeting.id,
)
# Add consent denial
await meeting_consent_controller.upsert(
MeetingConsent(
meeting_id=meeting.id,
user_id="test-user-3",
consent_given=False,
consent_timestamp=datetime.now(timezone.utc),
)
)
# Mock get_transcripts_storage (master credentials with bucket override)
with patch(
"reflector.pipelines.main_live_pipeline.get_transcripts_storage"
) as mock_get_transcripts_storage:
mock_master_storage = MagicMock()
mock_master_storage.delete_file = AsyncMock()
mock_get_transcripts_storage.return_value = mock_master_storage
# Run cleanup
await cleanup_consent(transcript_id=transcript.id)
# Verify master storage was used with bucket override
assert mock_master_storage.delete_file.call_count == 1
call_args = mock_master_storage.delete_file.call_args
assert call_args[0][0] == recording.object_key
assert call_args[1].get("bucket") == "test-bucket" # Verify bucket override!
@pytest.mark.asyncio
async def test_consent_cleanup_partial_failure_doesnt_mark_deleted():
room = await rooms_controller.add(
name="Test Room 4",
user_id="test-user",
zulip_auto_post=False,
zulip_stream="",
zulip_topic="",
is_locked=False,
room_mode="normal",
recording_type="cloud",
recording_trigger="automatic",
is_shared=False,
platform="daily",
)
# Create meeting
meeting = await meetings_controller.create(
id="test-multitrack-meeting-4",
room_name="test-room-20250101120003",
room_url="https://test.daily.co/test-room-4",
host_room_url="https://test.daily.co/test-room-4",
start_date=datetime.now(timezone.utc),
end_date=datetime.now(timezone.utc),
room=room,
)
track_keys = [
"recordings/test-room-20250101120003/track-0.webm",
"recordings/test-room-20250101120003/track-1.webm",
"recordings/test-room-20250101120003/track-2.webm",
]
recording = await recordings_controller.create(
Recording(
bucket_name="test-bucket",
object_key="recordings/test-room-20250101120003",
recorded_at=datetime.now(timezone.utc),
meeting_id=meeting.id,
track_keys=track_keys,
)
)
# Create transcript
transcript = await transcripts_controller.add(
name="Test Partial Failure Transcript",
source_kind=SourceKind.ROOM,
recording_id=recording.id,
meeting_id=meeting.id,
)
# Add consent denial
await meeting_consent_controller.upsert(
MeetingConsent(
meeting_id=meeting.id,
user_id="test-user-4",
consent_given=False,
consent_timestamp=datetime.now(timezone.utc),
)
)
# Mock get_transcripts_storage (master credentials with bucket override) with partial failure
with patch(
"reflector.pipelines.main_live_pipeline.get_transcripts_storage"
) as mock_get_transcripts_storage:
mock_master_storage = MagicMock()
call_count = 0
async def delete_side_effect(key, bucket=None):
nonlocal call_count
call_count += 1
if call_count == 2:
raise Exception("S3 deletion failed")
mock_master_storage.delete_file = AsyncMock(side_effect=delete_side_effect)
mock_get_transcripts_storage.return_value = mock_master_storage
await cleanup_consent(transcript_id=transcript.id)
# Verify master storage was called with bucket override
assert mock_master_storage.delete_file.call_count == 3
updated_transcript = await transcripts_controller.get_by_id(transcript.id)
assert (
updated_transcript.audio_deleted is None
or updated_transcript.audio_deleted is False
)