selfhostyourtech/reflector
2
0
Fork 0
mirror of https://github.com/Monadical-SAS/reflector.git synced 2025-12-21 04:39:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: dailyco poll (#730)

Browse Source 
* dailyco api module (no-mistakes)

* daily co library self-review

* uncurse

* self-review: daily resource leak, uniform types, enable_recording bomb, daily custom error, video_platforms/daily typing, daily timestamp dry

* dailyco docs parser

* phase 1-2 of daily poll

* dailyco poll (no-mistakes)

* poll docs

* fix tests

* forgotten utils file

* remove generated daily docs

* pr comments

* dailyco poll pr review and self-review

* daily recording poll api fix

* daily recording poll api fix

* review

* review

* fix tests

---------

Co-authored-by: Igor Loskutov <igor.loskutoff@gmail.com>
This commit is contained in:
Igor Monadical
2025-11-24 22:24:03 -05:00
committed by GitHub
parent 11731c9d38
commit 8e438ca285
21 changed files with 1529 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/tests/mocks/mock_platform.py
View File

@@ -64,12 +64,6 @@ class MockPlatformClient(VideoPlatformClient):
)
]
async def delete_room(self, room_name: str) -> bool:
if room_name in self._rooms:
self._rooms[room_name]["is_active"] = False
return True
return False
async def upload_logo(self, room_name: str, logo_path: str) -> bool:
if room_name in self._rooms:
self._rooms[room_name]["logo_path"] = logo_path

466
server/tests/test_daily_room_presence_polling.py Normal file
View File

@@ -0,0 +1,466 @@
"""Tests for Daily.co room presence polling functionality.
TDD tests for Task 3.2: Room Presence Polling
- Query Daily.co API for current room participants
- Reconcile with DB sessions (add missing, close stale)
- Update meeting.num_clients if different
- Use batch operations for efficiency
"""
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, patch
import pytest
from reflector.dailyco_api.responses import (
RoomPresenceParticipant,
RoomPresenceResponse,
)
from reflector.db.daily_participant_sessions import DailyParticipantSession
from reflector.db.meetings import Meeting
from reflector.worker.process import poll_daily_room_presence
@pytest.fixture
def mock_meeting():
"""Mock meeting with Daily.co room."""
return Meeting(
id="meeting-123",
room_id="room-456",
room_name="test-room-20251118120000",
room_url="https://daily.co/test-room-20251118120000",
host_room_url="https://daily.co/test-room-20251118120000?t=host-token",
platform="daily",
num_clients=2,
is_active=True,
start_date=datetime.now(timezone.utc),
end_date=datetime.now(timezone.utc),
)
@pytest.fixture
def mock_api_participants():
"""Mock Daily.co API presence response."""
now = datetime.now(timezone.utc)
return RoomPresenceResponse(
total_count=2,
data=[
RoomPresenceParticipant(
room="test-room-20251118120000",
id="participant-1",
userName="Alice",
userId="user-alice",
joinTime=(now - timedelta(minutes=10)).isoformat(),
duration=600,
),
RoomPresenceParticipant(
room="test-room-20251118120000",
id="participant-2",
userName="Bob",
userId="user-bob",
joinTime=(now - timedelta(minutes=5)).isoformat(),
duration=300,
),
],
)
@pytest.mark.asyncio
@patch("reflector.worker.process.meetings_controller.get_by_id")
@patch("reflector.worker.process.create_platform_client")
@patch(
"reflector.worker.process.daily_participant_sessions_controller.get_all_sessions_for_meeting"
)
@patch(
"reflector.worker.process.daily_participant_sessions_controller.batch_upsert_sessions"
)
async def test_poll_presence_adds_missing_sessions(
mock_batch_upsert,
mock_get_sessions,
mock_create_client,
mock_get_by_id,
mock_meeting,
mock_api_participants,
):
"""Test that polling creates sessions for participants not in DB."""
mock_get_by_id.return_value = mock_meeting
mock_daily_client = AsyncMock()
mock_daily_client.get_room_presence = AsyncMock(return_value=mock_api_participants)
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
mock_get_sessions.return_value = {}
mock_batch_upsert.return_value = None
await poll_daily_room_presence(mock_meeting.id)
assert mock_batch_upsert.call_count == 1
sessions = mock_batch_upsert.call_args.args[0]
assert len(sessions) == 2
session_ids = {s.session_id for s in sessions}
assert session_ids == {"participant-1", "participant-2"}
@pytest.mark.asyncio
@patch("reflector.worker.process.meetings_controller.get_by_id")
@patch("reflector.worker.process.create_platform_client")
@patch(
"reflector.worker.process.daily_participant_sessions_controller.get_all_sessions_for_meeting"
)
@patch(
"reflector.worker.process.daily_participant_sessions_controller.batch_upsert_sessions"
)
@patch(
"reflector.worker.process.daily_participant_sessions_controller.batch_close_sessions"
)
async def test_poll_presence_closes_stale_sessions(
mock_batch_close,
mock_batch_upsert,
mock_get_sessions,
mock_create_client,
mock_get_by_id,
mock_meeting,
mock_api_participants,
):
"""Test that polling closes sessions for participants no longer in room."""
mock_get_by_id.return_value = mock_meeting
mock_daily_client = AsyncMock()
mock_daily_client.get_room_presence = AsyncMock(return_value=mock_api_participants)
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
now = datetime.now(timezone.utc)
mock_get_sessions.return_value = {
"participant-1": DailyParticipantSession(
id=f"meeting-123:participant-1",
meeting_id="meeting-123",
room_id="room-456",
session_id="participant-1",
user_id="user-alice",
user_name="Alice",
joined_at=now,
left_at=None,
),
"participant-stale": DailyParticipantSession(
id=f"meeting-123:participant-stale",
meeting_id="meeting-123",
room_id="room-456",
session_id="participant-stale",
user_id="user-stale",
user_name="Stale User",
joined_at=now - timedelta(seconds=120), # Joined 2 minutes ago
left_at=None,
),
}
await poll_daily_room_presence(mock_meeting.id)
assert mock_batch_close.call_count == 1
composite_ids = mock_batch_close.call_args.args[0]
left_at = mock_batch_close.call_args.kwargs["left_at"]
assert len(composite_ids) == 1
assert "meeting-123:participant-stale" in composite_ids
assert left_at is not None
@pytest.mark.asyncio
@patch("reflector.worker.process.meetings_controller.get_by_id")
@patch("reflector.worker.process.create_platform_client")
@patch(
"reflector.worker.process.daily_participant_sessions_controller.get_all_sessions_for_meeting"
)
@patch(
"reflector.worker.process.daily_participant_sessions_controller.batch_upsert_sessions"
)
@patch("reflector.worker.process.meetings_controller.update_meeting")
async def test_poll_presence_updates_num_clients(
mock_update_meeting,
mock_batch_upsert,
mock_get_sessions,
mock_create_client,
mock_get_by_id,
mock_meeting,
mock_api_participants,
):
"""Test that polling updates num_clients when different from API."""
meeting_with_wrong_count = mock_meeting
meeting_with_wrong_count.num_clients = 5
mock_get_by_id.return_value = meeting_with_wrong_count
mock_daily_client = AsyncMock()
mock_daily_client.get_room_presence = AsyncMock(return_value=mock_api_participants)
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
mock_get_sessions.return_value = {}
mock_batch_upsert.return_value = None
await poll_daily_room_presence(meeting_with_wrong_count.id)
assert mock_update_meeting.call_count == 1
assert mock_update_meeting.call_args.kwargs["num_clients"] == 2
@pytest.mark.asyncio
@patch("reflector.worker.process.meetings_controller.get_by_id")
@patch("reflector.worker.process.create_platform_client")
@patch(
"reflector.worker.process.daily_participant_sessions_controller.get_all_sessions_for_meeting"
)
async def test_poll_presence_no_changes_if_synced(
mock_get_sessions,
mock_create_client,
mock_get_by_id,
mock_meeting,
mock_api_participants,
):
"""Test that polling skips updates when DB already synced with API."""
mock_get_by_id.return_value = mock_meeting
mock_daily_client = AsyncMock()
mock_daily_client.get_room_presence = AsyncMock(return_value=mock_api_participants)
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
now = datetime.now(timezone.utc)
mock_get_sessions.return_value = {
"participant-1": DailyParticipantSession(
id=f"meeting-123:participant-1",
meeting_id="meeting-123",
room_id="room-456",
session_id="participant-1",
user_id="user-alice",
user_name="Alice",
joined_at=now,
left_at=None,
),
"participant-2": DailyParticipantSession(
id=f"meeting-123:participant-2",
meeting_id="meeting-123",
room_id="room-456",
session_id="participant-2",
user_id="user-bob",
user_name="Bob",
joined_at=now,
left_at=None,
),
}
await poll_daily_room_presence(mock_meeting.id)
@pytest.mark.asyncio
@patch("reflector.worker.process.meetings_controller.get_by_id")
@patch("reflector.worker.process.create_platform_client")
@patch(
"reflector.worker.process.daily_participant_sessions_controller.get_all_sessions_for_meeting"
)
@patch(
"reflector.worker.process.daily_participant_sessions_controller.batch_upsert_sessions"
)
@patch(
"reflector.worker.process.daily_participant_sessions_controller.batch_close_sessions"
)
async def test_poll_presence_mixed_add_and_remove(
mock_batch_close,
mock_batch_upsert,
mock_get_sessions,
mock_create_client,
mock_get_by_id,
mock_meeting,
):
"""Test that polling handles simultaneous joins and leaves in single poll."""
mock_get_by_id.return_value = mock_meeting
now = datetime.now(timezone.utc)
# API returns: participant-1 and participant-3 (new)
api_response = RoomPresenceResponse(
total_count=2,
data=[
RoomPresenceParticipant(
room="test-room-20251118120000",
id="participant-1",
userName="Alice",
userId="user-alice",
joinTime=(now - timedelta(minutes=10)).isoformat(),
duration=600,
),
RoomPresenceParticipant(
room="test-room-20251118120000",
id="participant-3",
userName="Charlie",
userId="user-charlie",
joinTime=now.isoformat(),
duration=0,
),
],
)
mock_daily_client = AsyncMock()
mock_daily_client.get_room_presence = AsyncMock(return_value=api_response)
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
# DB has: participant-1 and participant-2 (left but not in API)
mock_get_sessions.return_value = {
"participant-1": DailyParticipantSession(
id=f"meeting-123:participant-1",
meeting_id="meeting-123",
room_id="room-456",
session_id="participant-1",
user_id="user-alice",
user_name="Alice",
joined_at=now - timedelta(minutes=10),
left_at=None,
),
"participant-2": DailyParticipantSession(
id=f"meeting-123:participant-2",
meeting_id="meeting-123",
room_id="room-456",
session_id="participant-2",
user_id="user-bob",
user_name="Bob",
joined_at=now - timedelta(minutes=5),
left_at=None,
),
}
mock_batch_upsert.return_value = None
mock_batch_close.return_value = None
await poll_daily_room_presence(mock_meeting.id)
# Verify participant-3 was added (missing in DB)
assert mock_batch_upsert.call_count == 1
sessions_added = mock_batch_upsert.call_args.args[0]
assert len(sessions_added) == 1
assert sessions_added[0].session_id == "participant-3"
assert sessions_added[0].user_name == "Charlie"
# Verify participant-2 was closed (stale in DB)
assert mock_batch_close.call_count == 1
composite_ids = mock_batch_close.call_args.args[0]
assert len(composite_ids) == 1
assert "meeting-123:participant-2" in composite_ids
@pytest.mark.asyncio
@patch("reflector.worker.process.meetings_controller.get_by_id")
@patch("reflector.worker.process.create_platform_client")
async def test_poll_presence_handles_api_error(
mock_create_client,
mock_get_by_id,
mock_meeting,
):
"""Test that polling handles Daily.co API errors gracefully."""
mock_get_by_id.return_value = mock_meeting
mock_daily_client = AsyncMock()
mock_daily_client.get_room_presence = AsyncMock(side_effect=Exception("API error"))
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
await poll_daily_room_presence(mock_meeting.id)
@pytest.mark.asyncio
@patch("reflector.worker.process.meetings_controller.get_by_id")
@patch("reflector.worker.process.create_platform_client")
@patch(
"reflector.worker.process.daily_participant_sessions_controller.get_all_sessions_for_meeting"
)
@patch(
"reflector.worker.process.daily_participant_sessions_controller.batch_close_sessions"
)
async def test_poll_presence_closes_all_when_room_empty(
mock_batch_close,
mock_get_sessions,
mock_create_client,
mock_get_by_id,
mock_meeting,
):
"""Test that polling closes all sessions when room is empty."""
mock_get_by_id.return_value = mock_meeting
mock_daily_client = AsyncMock()
mock_daily_client.get_room_presence = AsyncMock(
return_value=RoomPresenceResponse(total_count=0, data=[])
)
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
now = datetime.now(timezone.utc)
mock_get_sessions.return_value = {
"participant-1": DailyParticipantSession(
id=f"meeting-123:participant-1",
meeting_id="meeting-123",
room_id="room-456",
session_id="participant-1",
user_id="user-alice",
user_name="Alice",
joined_at=now
- timedelta(seconds=120), # Joined 2 minutes ago (beyond grace period)
left_at=None,
),
}
await poll_daily_room_presence(mock_meeting.id)
assert mock_batch_close.call_count == 1
composite_ids = mock_batch_close.call_args.args[0]
left_at = mock_batch_close.call_args.kwargs["left_at"]
assert len(composite_ids) == 1
assert "meeting-123:participant-1" in composite_ids
assert left_at is not None
@pytest.mark.asyncio
@patch("reflector.worker.process.RedisAsyncLock")
@patch("reflector.worker.process.meetings_controller.get_by_id")
@patch("reflector.worker.process.create_platform_client")
async def test_poll_presence_skips_if_locked(
mock_create_client,
mock_get_by_id,
mock_redis_lock_class,
mock_meeting,
):
"""Test that concurrent polling is prevented by Redis lock."""
mock_get_by_id.return_value = mock_meeting
# Mock the RedisAsyncLock to simulate lock not acquired
mock_lock_instance = AsyncMock()
mock_lock_instance.acquired = False # Lock not acquired
mock_lock_instance.__aenter__ = AsyncMock(return_value=mock_lock_instance)
mock_lock_instance.__aexit__ = AsyncMock()
mock_redis_lock_class.return_value = mock_lock_instance
mock_daily_client = AsyncMock()
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
await poll_daily_room_presence(mock_meeting.id)
# Verify RedisAsyncLock was instantiated
assert mock_redis_lock_class.call_count == 1
# Verify get_room_presence was NOT called (lock not acquired, so function returned early)
assert mock_daily_client.get_room_presence.call_count == 0

193
server/tests/test_poll_daily_recordings.py Normal file
View File

@@ -0,0 +1,193 @@
"""Tests for poll_daily_recordings task."""
from datetime import datetime, timedelta, timezone
from unittest.mock import AsyncMock, patch
import pytest
from reflector.dailyco_api.responses import RecordingResponse
from reflector.dailyco_api.webhooks import DailyTrack
# Import the unwrapped async function for testing
# The function is decorated with @shared_task and @asynctask,
# but we need to test the underlying async implementation
def _get_poll_daily_recordings_fn():
"""Get the underlying async function without Celery/asynctask decorators."""
from reflector.worker import process
# Access the actual async function before decorators
fn = process.poll_daily_recordings
# Get through both decorator layers
if hasattr(fn, "__wrapped__"):
fn = fn.__wrapped__
if hasattr(fn, "__wrapped__"):
fn = fn.__wrapped__
return fn
@pytest.fixture
def mock_recording_response():
"""Mock Daily.co API recording response with tracks."""
now = datetime.now(timezone.utc)
return [
RecordingResponse(
id="rec-123",
room_name="test-room-20251118120000",
start_ts=int((now - timedelta(hours=1)).timestamp()),
status="finished",
max_participants=2,
duration=3600,
share_token="share-token-123",
tracks=[
DailyTrack(type="audio", s3Key="track1.webm", size=1024),
DailyTrack(type="audio", s3Key="track2.webm", size=2048),
],
),
RecordingResponse(
id="rec-456",
room_name="test-room-20251118130000",
start_ts=int((now - timedelta(hours=2)).timestamp()),
status="finished",
max_participants=3,
duration=7200,
share_token="share-token-456",
tracks=[
DailyTrack(type="audio", s3Key="track1.webm", size=1024),
],
),
]
@pytest.mark.asyncio
@patch("reflector.worker.process.settings")
@patch("reflector.worker.process.create_platform_client")
@patch("reflector.worker.process.recordings_controller.get_by_ids")
@patch("reflector.worker.process.process_multitrack_recording.delay")
async def test_poll_daily_recordings_processes_missing_recordings(
mock_process_delay,
mock_get_recordings,
mock_create_client,
mock_settings,
mock_recording_response,
):
"""Test that poll_daily_recordings queues processing for recordings not in DB."""
mock_settings.DAILYCO_STORAGE_AWS_BUCKET_NAME = "test-bucket"
# Mock Daily.co API client
mock_daily_client = AsyncMock()
mock_daily_client.list_recordings = AsyncMock(return_value=mock_recording_response)
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
# Mock DB controller - no existing recordings
mock_get_recordings.return_value = []
# Execute - call the unwrapped async function
poll_fn = _get_poll_daily_recordings_fn()
await poll_fn()
# Verify Daily.co API was called without time parameters (uses default limit=100)
assert mock_daily_client.list_recordings.call_count == 1
call_kwargs = mock_daily_client.list_recordings.call_args.kwargs
# Should not have time-based parameters (uses cursor-based pagination)
assert "start_time" not in call_kwargs
assert "end_time" not in call_kwargs
# Verify processing was queued for both missing recordings
assert mock_process_delay.call_count == 2
# Verify the processing calls have correct parameters
calls = mock_process_delay.call_args_list
assert calls[0].kwargs["bucket_name"] == "test-bucket"
assert calls[0].kwargs["recording_id"] == "rec-123"
assert calls[0].kwargs["daily_room_name"] == "test-room-20251118120000"
assert calls[0].kwargs["track_keys"] == ["track1.webm", "track2.webm"]
assert calls[1].kwargs["bucket_name"] == "test-bucket"
assert calls[1].kwargs["recording_id"] == "rec-456"
assert calls[1].kwargs["daily_room_name"] == "test-room-20251118130000"
assert calls[1].kwargs["track_keys"] == ["track1.webm"]
@pytest.mark.asyncio
@patch("reflector.worker.process.settings")
@patch("reflector.worker.process.create_platform_client")
@patch("reflector.worker.process.recordings_controller.get_by_ids")
@patch("reflector.worker.process.process_multitrack_recording.delay")
async def test_poll_daily_recordings_skips_existing_recordings(
mock_process_delay,
mock_get_recordings,
mock_create_client,
mock_settings,
mock_recording_response,
):
"""Test that poll_daily_recordings skips recordings already in DB."""
mock_settings.DAILYCO_STORAGE_AWS_BUCKET_NAME = "test-bucket"
# Mock Daily.co API client
mock_daily_client = AsyncMock()
mock_daily_client.list_recordings = AsyncMock(return_value=mock_recording_response)
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
# Mock DB controller - all recordings already exist
from reflector.db.recordings import Recording
mock_get_recordings.return_value = [
Recording(
id="rec-123",
bucket_name="test-bucket",
object_key="",
recorded_at=datetime.now(timezone.utc),
meeting_id="meeting-1",
),
Recording(
id="rec-456",
bucket_name="test-bucket",
object_key="",
recorded_at=datetime.now(timezone.utc),
meeting_id="meeting-1",
),
]
# Execute - call the unwrapped async function
poll_fn = _get_poll_daily_recordings_fn()
await poll_fn()
# Verify Daily.co API was called
assert mock_daily_client.list_recordings.call_count == 1
# Verify NO processing was queued (all recordings already exist)
assert mock_process_delay.call_count == 0
@pytest.mark.asyncio
@patch("reflector.worker.process.settings")
@patch("reflector.worker.process.create_platform_client")
async def test_poll_daily_recordings_skips_when_bucket_not_configured(
mock_create_client,
mock_settings,
):
"""Test that poll_daily_recordings returns early when bucket is not configured."""
# No bucket configured
mock_settings.DAILYCO_STORAGE_AWS_BUCKET_NAME = None
# Mock should not be called
mock_daily_client = AsyncMock()
mock_create_client.return_value.__aenter__ = AsyncMock(
return_value=mock_daily_client
)
mock_create_client.return_value.__aexit__ = AsyncMock()
# Execute - call the unwrapped async function
poll_fn = _get_poll_daily_recordings_fn()
await poll_fn()
# Verify API was never called
mock_daily_client.list_recordings.assert_not_called()

49
server/tests/test_utils_daily.py
View File

@@ -1,6 +1,6 @@
import pytest
from reflector.utils.daily import extract_base_room_name
from reflector.utils.daily import extract_base_room_name, parse_daily_recording_filename
@pytest.mark.parametrize(
@@ -15,3 +15,50 @@ from reflector.utils.daily import extract_base_room_name
)
def test_extract_base_room_name(daily_room_name, expected):
assert extract_base_room_name(daily_room_name) == expected
@pytest.mark.parametrize(
"filename,expected_recording_ts,expected_participant_id,expected_track_ts",
[
(
"1763152299562-12f0b87c-97d4-4dd3-a65c-cee1f854a79c-cam-audio-1763152314582",
1763152299562,
"12f0b87c-97d4-4dd3-a65c-cee1f854a79c",
1763152314582,
),
(
"1760988935484-52f7f48b-fbab-431f-9a50-87b9abfc8255-cam-audio-1760988935922",
1760988935484,
"52f7f48b-fbab-431f-9a50-87b9abfc8255",
1760988935922,
),
(
"1760988935484-a37c35e3-6f8e-4274-a482-e9d0f102a732-cam-audio-1760988943823",
1760988935484,
"a37c35e3-6f8e-4274-a482-e9d0f102a732",
1760988943823,
),
(
"path/to/1763151171834-b6719a43-4481-483a-a8fc-2ae18b69283c-cam-audio-1763151180561",
1763151171834,
"b6719a43-4481-483a-a8fc-2ae18b69283c",
1763151180561,
),
],
)
def test_parse_daily_recording_filename(
filename, expected_recording_ts, expected_participant_id, expected_track_ts
):
result = parse_daily_recording_filename(filename)
assert result.recording_start_ts == expected_recording_ts
assert result.participant_id == expected_participant_id
assert result.track_start_ts == expected_track_ts
def test_parse_daily_recording_filename_invalid():
with pytest.raises(ValueError, match="Invalid Daily.co recording filename"):
parse_daily_recording_filename("invalid-filename")
with pytest.raises(ValueError, match="Invalid Daily.co recording filename"):
parse_daily_recording_filename("123-not-a-uuid-cam-audio-456")