mirror of
https://github.com/Monadical-SAS/reflector.git
synced 2026-03-22 07:06:47 +00:00
Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cb1beae90d | ||
|
|
1e396ca0ca | ||
|
|
9a2f973a2e | ||
|
|
a9200d35bf | ||
|
|
5646319e96 | ||
|
|
d0472ebf5f | ||
|
|
628a6d735c | ||
|
|
37a1f01850 | ||
|
|
72dca7cacc | ||
|
|
4ae56b730a | ||
|
|
cf6e867cf1 | ||
|
|
183601a121 | ||
|
|
b53c8da398 |
139
.github/workflows/integration_tests.yml
vendored
Normal file
139
.github/workflows/integration_tests.yml
vendored
Normal file
@@ -0,0 +1,139 @@
|
||||
name: Integration Tests
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
llm_model:
|
||||
description: "LLM model name (overrides LLM_MODEL secret)"
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
integration:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Start infrastructure services
|
||||
working-directory: server/tests
|
||||
env:
|
||||
LLM_URL: ${{ secrets.LLM_URL }}
|
||||
LLM_MODEL: ${{ inputs.llm_model || secrets.LLM_MODEL }}
|
||||
LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
run: |
|
||||
docker compose -f docker-compose.integration.yml up -d --build postgres redis garage hatchet mock-daily
|
||||
|
||||
- name: Set up Garage bucket and keys
|
||||
working-directory: server/tests
|
||||
run: |
|
||||
GARAGE="docker compose -f docker-compose.integration.yml exec -T garage /garage"
|
||||
GARAGE_KEY_ID="GK0123456789abcdef01234567" # gitleaks:allow
|
||||
GARAGE_KEY_SECRET="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" # gitleaks:allow
|
||||
|
||||
echo "Waiting for Garage to be healthy..."
|
||||
for i in $(seq 1 60); do
|
||||
if $GARAGE stats &>/dev/null; then break; fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "Setting up Garage..."
|
||||
NODE_ID=$($GARAGE node id -q 2>&1 | tr -d '[:space:]')
|
||||
LAYOUT_STATUS=$($GARAGE layout show 2>&1 || true)
|
||||
if echo "$LAYOUT_STATUS" | grep -q "No nodes"; then
|
||||
$GARAGE layout assign "$NODE_ID" -c 1G -z dc1
|
||||
$GARAGE layout apply --version 1
|
||||
fi
|
||||
|
||||
$GARAGE bucket info reflector-media &>/dev/null || $GARAGE bucket create reflector-media
|
||||
if ! $GARAGE key info reflector-test &>/dev/null; then
|
||||
$GARAGE key import --yes "$GARAGE_KEY_ID" "$GARAGE_KEY_SECRET"
|
||||
$GARAGE key rename "$GARAGE_KEY_ID" reflector-test
|
||||
fi
|
||||
$GARAGE bucket allow reflector-media --read --write --key reflector-test
|
||||
|
||||
- name: Wait for Hatchet and generate API token
|
||||
working-directory: server/tests
|
||||
run: |
|
||||
echo "Waiting for Hatchet to be healthy..."
|
||||
for i in $(seq 1 90); do
|
||||
if docker compose -f docker-compose.integration.yml exec -T hatchet curl -sf http://localhost:8888/api/live &>/dev/null; then
|
||||
echo "Hatchet is ready."
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "Generating Hatchet API token..."
|
||||
HATCHET_OUTPUT=$(docker compose -f docker-compose.integration.yml exec -T hatchet \
|
||||
/hatchet-admin token create --config /config --name integration-test 2>&1)
|
||||
HATCHET_TOKEN=$(echo "$HATCHET_OUTPUT" | grep -o 'eyJ[A-Za-z0-9_.\-]*')
|
||||
if [ -z "$HATCHET_TOKEN" ]; then
|
||||
echo "ERROR: Failed to extract Hatchet JWT token"
|
||||
exit 1
|
||||
fi
|
||||
echo "HATCHET_CLIENT_TOKEN=${HATCHET_TOKEN}" >> $GITHUB_ENV
|
||||
|
||||
- name: Start backend services
|
||||
working-directory: server/tests
|
||||
env:
|
||||
LLM_URL: ${{ secrets.LLM_URL }}
|
||||
LLM_MODEL: ${{ inputs.llm_model || secrets.LLM_MODEL }}
|
||||
LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
||||
run: |
|
||||
# Export garage and hatchet credentials for backend services
|
||||
export GARAGE_KEY_ID="${{ env.GARAGE_KEY_ID }}"
|
||||
export GARAGE_KEY_SECRET="${{ env.GARAGE_KEY_SECRET }}"
|
||||
export HATCHET_CLIENT_TOKEN="${{ env.HATCHET_CLIENT_TOKEN }}"
|
||||
|
||||
docker compose -f docker-compose.integration.yml up -d \
|
||||
server worker hatchet-worker-cpu hatchet-worker-llm test-runner
|
||||
|
||||
- name: Wait for server health check
|
||||
working-directory: server/tests
|
||||
run: |
|
||||
echo "Waiting for server to be healthy..."
|
||||
for i in $(seq 1 60); do
|
||||
if docker compose -f docker-compose.integration.yml exec -T test-runner \
|
||||
curl -sf http://server:1250/health &>/dev/null; then
|
||||
echo "Server is ready."
|
||||
break
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
- name: Run DB migrations
|
||||
working-directory: server/tests
|
||||
run: |
|
||||
docker compose -f docker-compose.integration.yml exec -T server \
|
||||
uv run alembic upgrade head
|
||||
|
||||
- name: Run integration tests
|
||||
working-directory: server/tests
|
||||
run: |
|
||||
docker compose -f docker-compose.integration.yml exec -T test-runner \
|
||||
uv run pytest tests/integration/ -v -x
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
working-directory: server/tests
|
||||
run: |
|
||||
docker compose -f docker-compose.integration.yml logs --tail=500 > integration-logs.txt 2>&1
|
||||
|
||||
- name: Upload logs artifact
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: integration-logs
|
||||
path: server/tests/integration-logs.txt
|
||||
retention-days: 7
|
||||
|
||||
- name: Teardown
|
||||
if: always()
|
||||
working-directory: server/tests
|
||||
run: |
|
||||
docker compose -f docker-compose.integration.yml down -v --remove-orphans
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -25,3 +25,4 @@ www/.env.production
|
||||
opencode.json
|
||||
|
||||
vibedocs/
|
||||
server/tests/integration/logs/
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# See https://pre-commit.com for more information
|
||||
# See https://pre-commit.com/hooks.html for more hooks
|
||||
exclude: '(^uv\.lock$|pnpm-lock\.yaml$)'
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
|
||||
21
CHANGELOG.md
21
CHANGELOG.md
@@ -1,5 +1,26 @@
|
||||
# Changelog
|
||||
|
||||
## [0.39.0](https://github.com/GreyhavenHQ/reflector/compare/v0.38.2...v0.39.0) (2026-03-18)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* migrate file and live post-processing pipelines from Celery to Hatchet workflow engine ([#911](https://github.com/GreyhavenHQ/reflector/issues/911)) ([37a1f01](https://github.com/GreyhavenHQ/reflector/commit/37a1f0185057dd43b68df2b12bb08d3b18e28d34))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* integration tests runner in CI ([#919](https://github.com/GreyhavenHQ/reflector/issues/919)) ([1e396ca](https://github.com/GreyhavenHQ/reflector/commit/1e396ca0ca91bc9d2645ddfc63a1576469491faa))
|
||||
* latest vulns ([#915](https://github.com/GreyhavenHQ/reflector/issues/915)) ([a9200d3](https://github.com/GreyhavenHQ/reflector/commit/a9200d35bf856f65f24a4f34931ebe0d75ad0382))
|
||||
|
||||
## [0.38.2](https://github.com/GreyhavenHQ/reflector/compare/v0.38.1...v0.38.2) (2026-03-12)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add auth guards to prevent anonymous access to write endpoints in non-public mode ([#907](https://github.com/GreyhavenHQ/reflector/issues/907)) ([cf6e867](https://github.com/GreyhavenHQ/reflector/commit/cf6e867cf12c42411e5a7412f6ec44eee8351665))
|
||||
* add tests that check some of the issues are already fixed ([#905](https://github.com/GreyhavenHQ/reflector/issues/905)) ([b53c8da](https://github.com/GreyhavenHQ/reflector/commit/b53c8da3981c394bdab08504b45d25f62c35495a))
|
||||
|
||||
## [0.38.1](https://github.com/GreyhavenHQ/reflector/compare/v0.38.0...v0.38.1) (2026-03-06)
|
||||
|
||||
|
||||
|
||||
15
CLAUDE.md
15
CLAUDE.md
@@ -160,6 +160,21 @@ All endpoints prefixed `/v1/`:
|
||||
- **Frontend**: No current test suite - opportunities for Jest/React Testing Library
|
||||
- **Coverage**: Backend maintains test coverage reports in `htmlcov/`
|
||||
|
||||
### Integration Tests (DO NOT run unless explicitly asked)
|
||||
|
||||
There are end-to-end integration tests in `server/tests/integration/` that spin up the full stack (PostgreSQL, Redis, Hatchet, Garage, mock-daily, server, workers) via Docker Compose and exercise real processing pipelines. These tests are:
|
||||
|
||||
- `test_file_pipeline.py` — File upload → FilePipeline
|
||||
- `test_live_pipeline.py` — WebRTC stream → LivePostPipeline
|
||||
- `test_multitrack_pipeline.py` — Multitrack → DailyMultitrackPipeline
|
||||
|
||||
**Important:**
|
||||
- These tests are **excluded** from normal `uv run pytest` runs via `--ignore=tests/integration` in pyproject.toml.
|
||||
- Do **NOT** run them as part of verification, code review, or general testing unless the user explicitly asks.
|
||||
- They require Docker, external LLM credentials, and HuggingFace token — they cannot run in a regular test environment.
|
||||
- To run locally: `./scripts/run-integration-tests.sh` (requires env vars: `LLM_URL`, `LLM_API_KEY`, `HF_TOKEN`).
|
||||
- In CI: triggered manually via the "Integration Tests" GitHub Actions workflow (`workflow_dispatch`).
|
||||
|
||||
## GPU Processing
|
||||
|
||||
Modal.com integration for scalable ML processing:
|
||||
|
||||
@@ -51,6 +51,9 @@ services:
|
||||
HF_TOKEN: ${HF_TOKEN:-}
|
||||
# WebRTC: fixed UDP port range for ICE candidates (mapped above)
|
||||
WEBRTC_PORT_RANGE: "51000-51100"
|
||||
# Hatchet workflow engine (always-on for processing pipelines)
|
||||
HATCHET_CLIENT_SERVER_URL: ${HATCHET_CLIENT_SERVER_URL:-http://hatchet:8888}
|
||||
HATCHET_CLIENT_HOST_PORT: ${HATCHET_CLIENT_HOST_PORT:-hatchet:7077}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -75,6 +78,9 @@ services:
|
||||
CELERY_RESULT_BACKEND: redis://redis:6379/1
|
||||
# ML backend config comes from env_file (server/.env), set per-mode by setup script
|
||||
HF_TOKEN: ${HF_TOKEN:-}
|
||||
# Hatchet workflow engine (always-on for processing pipelines)
|
||||
HATCHET_CLIENT_SERVER_URL: ${HATCHET_CLIENT_SERVER_URL:-http://hatchet:8888}
|
||||
HATCHET_CLIENT_HOST_PORT: ${HATCHET_CLIENT_HOST_PORT:-hatchet:7077}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
@@ -126,6 +132,8 @@ services:
|
||||
redis:
|
||||
image: redis:7.2-alpine
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "6379:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 30s
|
||||
@@ -301,20 +309,20 @@ services:
|
||||
- server
|
||||
|
||||
# ===========================================================
|
||||
# Hatchet + Daily.co workers (optional — for Daily.co multitrack processing)
|
||||
# Auto-enabled when DAILY_API_KEY is configured in server/r
|
||||
# Hatchet workflow engine + workers
|
||||
# Required for all processing pipelines (file, live, Daily.co multitrack).
|
||||
# Always-on — every selfhosted deployment needs Hatchet.
|
||||
# ===========================================================
|
||||
|
||||
hatchet:
|
||||
image: ghcr.io/hatchet-dev/hatchet/hatchet-lite:latest
|
||||
profiles: [dailyco]
|
||||
restart: on-failure
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8888:8888"
|
||||
- "7078:7077"
|
||||
- "127.0.0.1:8888:8888"
|
||||
- "127.0.0.1:7078:7077"
|
||||
env_file:
|
||||
- ./.env.hatchet
|
||||
environment:
|
||||
@@ -363,7 +371,6 @@ services:
|
||||
context: ./server
|
||||
dockerfile: Dockerfile
|
||||
image: monadicalsas/reflector-backend:latest
|
||||
profiles: [dailyco]
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- ./server/.env
|
||||
|
||||
156
scripts/run-integration-tests.sh
Executable file
156
scripts/run-integration-tests.sh
Executable file
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Run integration tests locally.
|
||||
#
|
||||
# Spins up the full stack via Docker Compose, runs the three integration tests,
|
||||
# and tears everything down afterward.
|
||||
#
|
||||
# Required environment variables:
|
||||
# LLM_URL — OpenAI-compatible LLM endpoint (e.g. https://api.openai.com/v1)
|
||||
# LLM_API_KEY — API key for the LLM endpoint
|
||||
# HF_TOKEN — HuggingFace token for pyannote gated models
|
||||
#
|
||||
# Optional:
|
||||
# LLM_MODEL — Model name (default: qwen2.5:14b)
|
||||
#
|
||||
# Usage:
|
||||
# export LLM_URL="https://api.openai.com/v1"
|
||||
# export LLM_API_KEY="sk-..."
|
||||
# export HF_TOKEN="hf_..."
|
||||
# ./scripts/run-integration-tests.sh
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
COMPOSE_DIR="$REPO_ROOT/server/tests"
|
||||
COMPOSE_FILE="$COMPOSE_DIR/docker-compose.integration.yml"
|
||||
COMPOSE="docker compose -f $COMPOSE_FILE"
|
||||
|
||||
# ── Validate required env vars ──────────────────────────────────────────────
|
||||
for var in LLM_URL LLM_API_KEY HF_TOKEN; do
|
||||
if [[ -z "${!var:-}" ]]; then
|
||||
echo "ERROR: $var is not set. See script header for required env vars."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
export LLM_MODEL="${LLM_MODEL:-qwen2.5:14b}"
|
||||
|
||||
# ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
info() { echo -e "\n\033[1;34m▸ $*\033[0m"; }
|
||||
ok() { echo -e "\033[1;32m ✓ $*\033[0m"; }
|
||||
fail() { echo -e "\033[1;31m ✗ $*\033[0m"; }
|
||||
|
||||
wait_for() {
|
||||
local desc="$1" cmd="$2" max="${3:-60}"
|
||||
info "Waiting for $desc (up to ${max}s)..."
|
||||
for i in $(seq 1 "$max"); do
|
||||
if eval "$cmd" &>/dev/null; then
|
||||
ok "$desc is ready"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
fail "$desc did not become ready within ${max}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
info "Tearing down..."
|
||||
$COMPOSE down -v --remove-orphans 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Always tear down on exit
|
||||
trap cleanup EXIT
|
||||
|
||||
# ── Step 1: Build and start infrastructure ──────────────────────────────────
|
||||
info "Building and starting infrastructure services..."
|
||||
$COMPOSE up -d --build postgres redis garage hatchet mock-daily
|
||||
|
||||
# ── Step 2: Set up Garage (S3 bucket + keys) ───────────────────────────────
|
||||
wait_for "Garage" "$COMPOSE exec -T garage /garage stats" 60
|
||||
|
||||
info "Setting up Garage bucket and keys..."
|
||||
GARAGE="$COMPOSE exec -T garage /garage"
|
||||
|
||||
# Hardcoded test credentials — ephemeral containers, destroyed after tests
|
||||
export GARAGE_KEY_ID="GK0123456789abcdef01234567" # gitleaks:allow
|
||||
export GARAGE_KEY_SECRET="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" # gitleaks:allow
|
||||
|
||||
# Layout
|
||||
NODE_ID=$($GARAGE node id -q 2>&1 | tr -d '[:space:]')
|
||||
LAYOUT_STATUS=$($GARAGE layout show 2>&1 || true)
|
||||
if echo "$LAYOUT_STATUS" | grep -q "No nodes"; then
|
||||
$GARAGE layout assign "$NODE_ID" -c 1G -z dc1
|
||||
$GARAGE layout apply --version 1
|
||||
fi
|
||||
|
||||
# Bucket
|
||||
$GARAGE bucket info reflector-media >/dev/null 2>&1 || $GARAGE bucket create reflector-media
|
||||
|
||||
# Import key with known credentials
|
||||
if ! $GARAGE key info reflector-test >/dev/null 2>&1; then
|
||||
$GARAGE key import --yes "$GARAGE_KEY_ID" "$GARAGE_KEY_SECRET"
|
||||
$GARAGE key rename "$GARAGE_KEY_ID" reflector-test
|
||||
fi
|
||||
|
||||
# Permissions
|
||||
$GARAGE bucket allow reflector-media --read --write --key reflector-test
|
||||
|
||||
ok "Garage ready with hardcoded test credentials"
|
||||
|
||||
# ── Step 3: Generate Hatchet API token ──────────────────────────────────────
|
||||
wait_for "Hatchet" "$COMPOSE exec -T hatchet curl -sf http://localhost:8888/api/live" 90
|
||||
|
||||
info "Generating Hatchet API token..."
|
||||
HATCHET_TOKEN_OUTPUT=$($COMPOSE exec -T hatchet /hatchet-admin token create --config /config --name local-test 2>&1)
|
||||
export HATCHET_CLIENT_TOKEN=$(echo "$HATCHET_TOKEN_OUTPUT" | grep -o 'eyJ[A-Za-z0-9_.\-]*')
|
||||
|
||||
if [[ -z "$HATCHET_CLIENT_TOKEN" ]]; then
|
||||
fail "Failed to extract Hatchet token (JWT not found in output)"
|
||||
echo " Output was: $HATCHET_TOKEN_OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
ok "Hatchet token generated"
|
||||
|
||||
# ── Step 4: Start backend services ──────────────────────────────────────────
|
||||
info "Starting backend services..."
|
||||
$COMPOSE up -d server worker hatchet-worker-cpu hatchet-worker-llm test-runner
|
||||
|
||||
# ── Step 5: Wait for server + run migrations ────────────────────────────────
|
||||
wait_for "Server" "$COMPOSE exec -T test-runner curl -sf http://server:1250/health" 60
|
||||
|
||||
info "Running database migrations..."
|
||||
$COMPOSE exec -T server uv run alembic upgrade head
|
||||
ok "Migrations applied"
|
||||
|
||||
# ── Step 6: Run integration tests ───────────────────────────────────────────
|
||||
info "Running integration tests..."
|
||||
echo ""
|
||||
|
||||
LOGS_DIR="$COMPOSE_DIR/integration/logs"
|
||||
mkdir -p "$LOGS_DIR"
|
||||
RUN_TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
TEST_LOG="$LOGS_DIR/$RUN_TIMESTAMP.txt"
|
||||
|
||||
if $COMPOSE exec -T test-runner uv run pytest tests/integration/ -v -x 2>&1 | tee "$TEST_LOG.pytest"; then
|
||||
echo ""
|
||||
ok "All integration tests passed!"
|
||||
EXIT_CODE=0
|
||||
else
|
||||
echo ""
|
||||
fail "Integration tests failed!"
|
||||
EXIT_CODE=1
|
||||
fi
|
||||
|
||||
# Always collect service logs + test output into a single file
|
||||
info "Collecting logs..."
|
||||
$COMPOSE logs --tail=500 > "$TEST_LOG" 2>&1
|
||||
echo -e "\n\n=== PYTEST OUTPUT ===\n" >> "$TEST_LOG"
|
||||
cat "$TEST_LOG.pytest" >> "$TEST_LOG" 2>/dev/null
|
||||
rm -f "$TEST_LOG.pytest"
|
||||
echo " Logs saved to: server/tests/integration/logs/$RUN_TIMESTAMP.txt"
|
||||
|
||||
# cleanup runs via trap
|
||||
exit $EXIT_CODE
|
||||
@@ -261,9 +261,11 @@ if [[ -z "$MODEL_MODE" ]]; then
|
||||
fi
|
||||
|
||||
# Build profiles list — one profile per feature
|
||||
# Only --gpu needs a compose profile; --cpu and --hosted use in-process/remote backends
|
||||
# Hatchet + hatchet-worker-llm are always-on (no profile needed).
|
||||
# gpu/cpu profiles only control the ML container (transcription service).
|
||||
COMPOSE_PROFILES=()
|
||||
[[ "$MODEL_MODE" == "gpu" ]] && COMPOSE_PROFILES+=("gpu")
|
||||
[[ "$MODEL_MODE" == "cpu" ]] && COMPOSE_PROFILES+=("cpu")
|
||||
[[ -n "$OLLAMA_MODE" ]] && COMPOSE_PROFILES+=("$OLLAMA_MODE")
|
||||
[[ "$USE_GARAGE" == "true" ]] && COMPOSE_PROFILES+=("garage")
|
||||
[[ "$USE_CADDY" == "true" ]] && COMPOSE_PROFILES+=("caddy")
|
||||
@@ -557,12 +559,10 @@ step_server_env() {
|
||||
ok "CPU mode — file processing timeouts set to 3600s (1 hour)"
|
||||
fi
|
||||
|
||||
# If Daily.co is manually configured, ensure Hatchet connectivity vars are set
|
||||
if env_has_key "$SERVER_ENV" "DAILY_API_KEY" && [[ -n "$(env_get "$SERVER_ENV" "DAILY_API_KEY")" ]]; then
|
||||
env_set "$SERVER_ENV" "HATCHET_CLIENT_SERVER_URL" "http://hatchet:8888"
|
||||
env_set "$SERVER_ENV" "HATCHET_CLIENT_HOST_PORT" "hatchet:7077"
|
||||
ok "Daily.co detected — Hatchet connectivity configured"
|
||||
fi
|
||||
# Hatchet is always required (file, live, and multitrack pipelines all use it)
|
||||
env_set "$SERVER_ENV" "HATCHET_CLIENT_SERVER_URL" "http://hatchet:8888"
|
||||
env_set "$SERVER_ENV" "HATCHET_CLIENT_HOST_PORT" "hatchet:7077"
|
||||
ok "Hatchet connectivity configured (workflow engine for processing pipelines)"
|
||||
|
||||
ok "server/.env ready"
|
||||
}
|
||||
@@ -886,15 +886,22 @@ step_services() {
|
||||
compose_cmd pull server web || warn "Pull failed — using cached images"
|
||||
fi
|
||||
|
||||
# Build hatchet workers if Daily.co is configured (same backend image)
|
||||
if [[ "$DAILY_DETECTED" == "true" ]] && [[ "$BUILD_IMAGES" == "true" ]]; then
|
||||
# Hatchet is always needed (all processing pipelines use it)
|
||||
local NEEDS_HATCHET=true
|
||||
|
||||
# Build hatchet workers if Hatchet is needed (same backend image)
|
||||
if [[ "$NEEDS_HATCHET" == "true" ]] && [[ "$BUILD_IMAGES" == "true" ]]; then
|
||||
info "Building Hatchet worker images..."
|
||||
compose_cmd build hatchet-worker-cpu hatchet-worker-llm
|
||||
if [[ "$DAILY_DETECTED" == "true" ]]; then
|
||||
compose_cmd build hatchet-worker-cpu hatchet-worker-llm
|
||||
else
|
||||
compose_cmd build hatchet-worker-llm
|
||||
fi
|
||||
ok "Hatchet worker images built"
|
||||
fi
|
||||
|
||||
# Ensure hatchet database exists before starting hatchet (init-hatchet-db.sql only runs on fresh postgres volumes)
|
||||
if [[ "$DAILY_DETECTED" == "true" ]]; then
|
||||
if [[ "$NEEDS_HATCHET" == "true" ]]; then
|
||||
info "Ensuring postgres is running for Hatchet database setup..."
|
||||
compose_cmd up -d postgres
|
||||
local pg_ready=false
|
||||
@@ -1049,24 +1056,22 @@ step_health() {
|
||||
fi
|
||||
fi
|
||||
|
||||
# Hatchet (if Daily.co detected)
|
||||
if [[ "$DAILY_DETECTED" == "true" ]]; then
|
||||
info "Waiting for Hatchet workflow engine..."
|
||||
local hatchet_ok=false
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8888/api/live > /dev/null 2>&1; then
|
||||
hatchet_ok=true
|
||||
break
|
||||
fi
|
||||
echo -ne "\r Waiting for Hatchet... ($i/60)"
|
||||
sleep 3
|
||||
done
|
||||
echo ""
|
||||
if [[ "$hatchet_ok" == "true" ]]; then
|
||||
ok "Hatchet workflow engine healthy"
|
||||
else
|
||||
warn "Hatchet not ready yet. Check: docker compose logs hatchet"
|
||||
# Hatchet (always-on)
|
||||
info "Waiting for Hatchet workflow engine..."
|
||||
local hatchet_ok=false
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://localhost:8888/api/live > /dev/null 2>&1; then
|
||||
hatchet_ok=true
|
||||
break
|
||||
fi
|
||||
echo -ne "\r Waiting for Hatchet... ($i/60)"
|
||||
sleep 3
|
||||
done
|
||||
echo ""
|
||||
if [[ "$hatchet_ok" == "true" ]]; then
|
||||
ok "Hatchet workflow engine healthy"
|
||||
else
|
||||
warn "Hatchet not ready yet. Check: docker compose logs hatchet"
|
||||
fi
|
||||
|
||||
# LLM warning for non-Ollama modes
|
||||
@@ -1087,12 +1092,10 @@ step_health() {
|
||||
}
|
||||
|
||||
# =========================================================
|
||||
# Step 8: Hatchet token generation (Daily.co only)
|
||||
# Step 8: Hatchet token generation (gpu/cpu/Daily.co)
|
||||
# =========================================================
|
||||
step_hatchet_token() {
|
||||
if [[ "$DAILY_DETECTED" != "true" ]]; then
|
||||
return
|
||||
fi
|
||||
# Hatchet is always required — no gating needed
|
||||
|
||||
# Skip if token already set
|
||||
if env_has_key "$SERVER_ENV" "HATCHET_CLIENT_TOKEN" && [[ -n "$(env_get "$SERVER_ENV" "HATCHET_CLIENT_TOKEN")" ]]; then
|
||||
@@ -1147,7 +1150,9 @@ step_hatchet_token() {
|
||||
|
||||
# Restart services that need the token
|
||||
info "Restarting services with new Hatchet token..."
|
||||
compose_cmd restart server worker hatchet-worker-cpu hatchet-worker-llm
|
||||
local restart_services="server worker hatchet-worker-llm"
|
||||
[[ "$DAILY_DETECTED" == "true" ]] && restart_services="$restart_services hatchet-worker-cpu"
|
||||
compose_cmd restart $restart_services
|
||||
ok "Services restarted with Hatchet token"
|
||||
}
|
||||
|
||||
@@ -1216,28 +1221,23 @@ main() {
|
||||
ok "Daily.co detected — enabling Hatchet workflow services"
|
||||
fi
|
||||
|
||||
# Generate .env.hatchet for hatchet dashboard config
|
||||
if [[ "$DAILY_DETECTED" == "true" ]]; then
|
||||
local hatchet_server_url hatchet_cookie_domain
|
||||
if [[ -n "$CUSTOM_DOMAIN" ]]; then
|
||||
hatchet_server_url="https://${CUSTOM_DOMAIN}:8888"
|
||||
hatchet_cookie_domain="$CUSTOM_DOMAIN"
|
||||
elif [[ -n "$PRIMARY_IP" ]]; then
|
||||
hatchet_server_url="http://${PRIMARY_IP}:8888"
|
||||
hatchet_cookie_domain="$PRIMARY_IP"
|
||||
else
|
||||
hatchet_server_url="http://localhost:8888"
|
||||
hatchet_cookie_domain="localhost"
|
||||
fi
|
||||
cat > "$ROOT_DIR/.env.hatchet" << EOF
|
||||
# Generate .env.hatchet for hatchet dashboard config (always needed)
|
||||
local hatchet_server_url hatchet_cookie_domain
|
||||
if [[ -n "$CUSTOM_DOMAIN" ]]; then
|
||||
hatchet_server_url="https://${CUSTOM_DOMAIN}:8888"
|
||||
hatchet_cookie_domain="$CUSTOM_DOMAIN"
|
||||
elif [[ -n "$PRIMARY_IP" ]]; then
|
||||
hatchet_server_url="http://${PRIMARY_IP}:8888"
|
||||
hatchet_cookie_domain="$PRIMARY_IP"
|
||||
else
|
||||
hatchet_server_url="http://localhost:8888"
|
||||
hatchet_cookie_domain="localhost"
|
||||
fi
|
||||
cat > "$ROOT_DIR/.env.hatchet" << EOF
|
||||
SERVER_URL=$hatchet_server_url
|
||||
SERVER_AUTH_COOKIE_DOMAIN=$hatchet_cookie_domain
|
||||
EOF
|
||||
ok "Generated .env.hatchet (dashboard URL=$hatchet_server_url)"
|
||||
else
|
||||
# Create empty .env.hatchet so compose doesn't fail if dailyco profile is ever activated manually
|
||||
touch "$ROOT_DIR/.env.hatchet"
|
||||
fi
|
||||
ok "Generated .env.hatchet (dashboard URL=$hatchet_server_url)"
|
||||
|
||||
step_www_env
|
||||
echo ""
|
||||
|
||||
@@ -116,9 +116,10 @@ source = ["reflector"]
|
||||
ENVIRONMENT = "pytest"
|
||||
DATABASE_URL = "postgresql://test_user:test_password@localhost:15432/reflector_test"
|
||||
AUTH_BACKEND = "jwt"
|
||||
HATCHET_CLIENT_TOKEN = "test-dummy-token"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
addopts = "-ra -q --disable-pytest-warnings --cov --cov-report html -v"
|
||||
addopts = "-ra -q --disable-pytest-warnings --cov --cov-report html -v --ignore=tests/integration"
|
||||
testpaths = ["tests"]
|
||||
asyncio_mode = "auto"
|
||||
markers = [
|
||||
|
||||
@@ -12,6 +12,7 @@ AccessTokenInfo = auth_module.AccessTokenInfo
|
||||
authenticated = auth_module.authenticated
|
||||
current_user = auth_module.current_user
|
||||
current_user_optional = auth_module.current_user_optional
|
||||
current_user_optional_if_public_mode = auth_module.current_user_optional_if_public_mode
|
||||
parse_ws_bearer_token = auth_module.parse_ws_bearer_token
|
||||
current_user_ws_optional = auth_module.current_user_ws_optional
|
||||
verify_raw_token = auth_module.verify_raw_token
|
||||
|
||||
@@ -129,6 +129,17 @@ async def current_user_optional(
|
||||
return await _authenticate_user(jwt_token, api_key, jwtauth)
|
||||
|
||||
|
||||
async def current_user_optional_if_public_mode(
|
||||
jwt_token: Annotated[Optional[str], Depends(oauth2_scheme)],
|
||||
api_key: Annotated[Optional[str], Depends(api_key_header)],
|
||||
jwtauth: JWTAuth = Depends(),
|
||||
) -> Optional[UserInfo]:
|
||||
user = await _authenticate_user(jwt_token, api_key, jwtauth)
|
||||
if user is None and not settings.PUBLIC_MODE:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
return user
|
||||
|
||||
|
||||
def parse_ws_bearer_token(
|
||||
websocket: "WebSocket",
|
||||
) -> tuple[Optional[str], Optional[str]]:
|
||||
|
||||
@@ -21,6 +21,11 @@ def current_user_optional():
|
||||
return None
|
||||
|
||||
|
||||
def current_user_optional_if_public_mode():
|
||||
# auth_none means no authentication at all — always public
|
||||
return None
|
||||
|
||||
|
||||
def parse_ws_bearer_token(websocket):
|
||||
return None, None
|
||||
|
||||
|
||||
@@ -150,6 +150,16 @@ async def current_user_optional(
|
||||
return await _authenticate_user(jwt_token, api_key)
|
||||
|
||||
|
||||
async def current_user_optional_if_public_mode(
|
||||
jwt_token: Annotated[Optional[str], Depends(oauth2_scheme)],
|
||||
api_key: Annotated[Optional[str], Depends(api_key_header)],
|
||||
) -> Optional[UserInfo]:
|
||||
user = await _authenticate_user(jwt_token, api_key)
|
||||
if user is None and not settings.PUBLIC_MODE:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
return user
|
||||
|
||||
|
||||
# --- WebSocket auth (same pattern as auth_jwt.py) ---
|
||||
def parse_ws_bearer_token(
|
||||
websocket: "WebSocket",
|
||||
|
||||
@@ -697,6 +697,18 @@ class TranscriptController:
|
||||
return False
|
||||
return user_id and transcript.user_id == user_id
|
||||
|
||||
@staticmethod
|
||||
def check_can_mutate(transcript: Transcript, user_id: str | None) -> None:
|
||||
"""
|
||||
Raises HTTP 403 if the user cannot mutate the transcript.
|
||||
|
||||
Policy:
|
||||
- Anonymous transcripts (user_id is None) are editable by anyone
|
||||
- Owned transcripts can only be mutated by their owner
|
||||
"""
|
||||
if transcript.user_id is not None and transcript.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
"""
|
||||
|
||||
@@ -26,6 +26,21 @@ class TaskName(StrEnum):
|
||||
DETECT_CHUNK_TOPIC = "detect_chunk_topic"
|
||||
GENERATE_DETAILED_SUMMARY = "generate_detailed_summary"
|
||||
|
||||
# File pipeline tasks
|
||||
EXTRACT_AUDIO = "extract_audio"
|
||||
UPLOAD_AUDIO = "upload_audio"
|
||||
TRANSCRIBE = "transcribe"
|
||||
DIARIZE = "diarize"
|
||||
ASSEMBLE_TRANSCRIPT = "assemble_transcript"
|
||||
GENERATE_SUMMARIES = "generate_summaries"
|
||||
|
||||
# Live post-processing pipeline tasks
|
||||
WAVEFORM = "waveform"
|
||||
CONVERT_MP3 = "convert_mp3"
|
||||
UPLOAD_MP3 = "upload_mp3"
|
||||
REMOVE_UPLOAD = "remove_upload"
|
||||
FINAL_SUMMARIES = "final_summaries"
|
||||
|
||||
|
||||
# Rate limit key for LLM API calls (shared across all LLM-calling tasks)
|
||||
LLM_RATE_LIMIT_KEY = "llm"
|
||||
|
||||
@@ -10,6 +10,8 @@ from reflector.hatchet.client import HatchetClientManager
|
||||
from reflector.hatchet.workflows.daily_multitrack_pipeline import (
|
||||
daily_multitrack_pipeline,
|
||||
)
|
||||
from reflector.hatchet.workflows.file_pipeline import file_pipeline
|
||||
from reflector.hatchet.workflows.live_post_pipeline import live_post_pipeline
|
||||
from reflector.hatchet.workflows.subject_processing import subject_workflow
|
||||
from reflector.hatchet.workflows.topic_chunk_processing import topic_chunk_workflow
|
||||
from reflector.hatchet.workflows.track_processing import track_workflow
|
||||
@@ -47,6 +49,8 @@ def main():
|
||||
},
|
||||
workflows=[
|
||||
daily_multitrack_pipeline,
|
||||
file_pipeline,
|
||||
live_post_pipeline,
|
||||
topic_chunk_workflow,
|
||||
subject_workflow,
|
||||
track_workflow,
|
||||
|
||||
@@ -307,7 +307,9 @@ async def get_recording(input: PipelineInput, ctx: Context) -> RecordingResult:
|
||||
ctx.log(
|
||||
f"get_recording: calling Daily.co API for recording_id={input.recording_id}..."
|
||||
)
|
||||
async with DailyApiClient(api_key=settings.DAILY_API_KEY) as client:
|
||||
async with DailyApiClient(
|
||||
api_key=settings.DAILY_API_KEY, base_url=settings.DAILY_API_URL
|
||||
) as client:
|
||||
recording = await client.get_recording(input.recording_id)
|
||||
ctx.log(f"get_recording: Daily.co API returned successfully")
|
||||
|
||||
@@ -374,7 +376,9 @@ async def get_participants(input: PipelineInput, ctx: Context) -> ParticipantsRe
|
||||
settings.DAILY_API_KEY, "DAILY_API_KEY is required"
|
||||
)
|
||||
|
||||
async with DailyApiClient(api_key=daily_api_key) as client:
|
||||
async with DailyApiClient(
|
||||
api_key=daily_api_key, base_url=settings.DAILY_API_URL
|
||||
) as client:
|
||||
participants = await client.get_meeting_participants(mtg_session_id)
|
||||
|
||||
id_to_name = {}
|
||||
|
||||
885
server/reflector/hatchet/workflows/file_pipeline.py
Normal file
885
server/reflector/hatchet/workflows/file_pipeline.py
Normal file
@@ -0,0 +1,885 @@
|
||||
"""
|
||||
Hatchet workflow: FilePipeline
|
||||
|
||||
Processing pipeline for file uploads and Whereby recordings.
|
||||
Orchestrates: extract audio → upload → transcribe/diarize/waveform (parallel)
|
||||
→ assemble → detect topics → title/summaries (parallel) → finalize
|
||||
→ cleanup consent → post zulip / send webhook.
|
||||
|
||||
Note: This file uses deferred imports (inside functions/tasks) intentionally.
|
||||
Hatchet workers run in forked processes; fresh imports per task ensure DB connections
|
||||
are not shared across forks, avoiding connection pooling issues.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from hatchet_sdk import Context
|
||||
from pydantic import BaseModel
|
||||
|
||||
from reflector.hatchet.broadcast import (
|
||||
append_event_and_broadcast,
|
||||
set_status_and_broadcast,
|
||||
)
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
from reflector.hatchet.constants import (
|
||||
TIMEOUT_HEAVY,
|
||||
TIMEOUT_MEDIUM,
|
||||
TIMEOUT_SHORT,
|
||||
TIMEOUT_TITLE,
|
||||
TaskName,
|
||||
)
|
||||
from reflector.hatchet.workflows.daily_multitrack_pipeline import (
|
||||
fresh_db_connection,
|
||||
set_workflow_error_status,
|
||||
with_error_handling,
|
||||
)
|
||||
from reflector.hatchet.workflows.models import (
|
||||
ConsentResult,
|
||||
TitleResult,
|
||||
TopicsResult,
|
||||
WaveformResult,
|
||||
WebhookResult,
|
||||
ZulipResult,
|
||||
)
|
||||
from reflector.logger import logger
|
||||
from reflector.pipelines import topic_processing
|
||||
from reflector.settings import settings
|
||||
from reflector.utils.audio_constants import WAVEFORM_SEGMENTS
|
||||
from reflector.utils.audio_waveform import get_audio_waveform
|
||||
|
||||
|
||||
class FilePipelineInput(BaseModel):
|
||||
transcript_id: str
|
||||
room_id: str | None = None
|
||||
|
||||
|
||||
# --- Result models specific to file pipeline ---
|
||||
|
||||
|
||||
class ExtractAudioResult(BaseModel):
|
||||
audio_path: str
|
||||
duration_ms: float = 0.0
|
||||
|
||||
|
||||
class UploadAudioResult(BaseModel):
|
||||
audio_url: str
|
||||
audio_path: str
|
||||
|
||||
|
||||
class TranscribeResult(BaseModel):
|
||||
words: list[dict]
|
||||
translation: str | None = None
|
||||
|
||||
|
||||
class DiarizeResult(BaseModel):
|
||||
diarization: list[dict] | None = None
|
||||
|
||||
|
||||
class AssembleTranscriptResult(BaseModel):
|
||||
assembled: bool
|
||||
|
||||
|
||||
class SummariesResult(BaseModel):
|
||||
generated: bool
|
||||
|
||||
|
||||
class FinalizeResult(BaseModel):
|
||||
status: str
|
||||
|
||||
|
||||
hatchet = HatchetClientManager.get_client()
|
||||
|
||||
file_pipeline = hatchet.workflow(name="FilePipeline", input_validator=FilePipelineInput)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.EXTRACT_AUDIO)
|
||||
async def extract_audio(input: FilePipelineInput, ctx: Context) -> ExtractAudioResult:
|
||||
"""Extract audio from upload file, convert to MP3."""
|
||||
ctx.log(f"extract_audio: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.transcripts import transcripts_controller # noqa: PLC0415
|
||||
|
||||
await set_status_and_broadcast(input.transcript_id, "processing", logger=logger)
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if not transcript:
|
||||
raise ValueError(f"Transcript {input.transcript_id} not found")
|
||||
|
||||
# Clear transcript as we're going to regenerate everything
|
||||
await transcripts_controller.update(
|
||||
transcript,
|
||||
{
|
||||
"events": [],
|
||||
"topics": [],
|
||||
},
|
||||
)
|
||||
|
||||
# Find upload file
|
||||
audio_file = next(transcript.data_path.glob("upload.*"), None)
|
||||
if not audio_file:
|
||||
audio_file = next(transcript.data_path.glob("audio.*"), None)
|
||||
if not audio_file:
|
||||
raise ValueError("No audio file found to process")
|
||||
|
||||
ctx.log(f"extract_audio: processing {audio_file}")
|
||||
|
||||
# Extract audio and write as MP3
|
||||
import av # noqa: PLC0415
|
||||
|
||||
from reflector.processors import AudioFileWriterProcessor # noqa: PLC0415
|
||||
|
||||
duration_ms_container = [0.0]
|
||||
|
||||
async def capture_duration(d):
|
||||
duration_ms_container[0] = d
|
||||
|
||||
mp3_writer = AudioFileWriterProcessor(
|
||||
path=transcript.audio_mp3_filename,
|
||||
on_duration=capture_duration,
|
||||
)
|
||||
input_container = av.open(str(audio_file))
|
||||
for frame in input_container.decode(audio=0):
|
||||
await mp3_writer.push(frame)
|
||||
await mp3_writer.flush()
|
||||
input_container.close()
|
||||
|
||||
duration_ms = duration_ms_container[0]
|
||||
audio_path = str(transcript.audio_mp3_filename)
|
||||
|
||||
# Persist duration to database and broadcast to websocket clients
|
||||
from reflector.db.transcripts import TranscriptDuration # noqa: PLC0415
|
||||
from reflector.db.transcripts import transcripts_controller as tc
|
||||
|
||||
await tc.update(transcript, {"duration": duration_ms})
|
||||
await append_event_and_broadcast(
|
||||
input.transcript_id,
|
||||
transcript,
|
||||
"DURATION",
|
||||
TranscriptDuration(duration=duration_ms),
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
ctx.log(f"extract_audio complete: {audio_path}, duration={duration_ms}ms")
|
||||
return ExtractAudioResult(audio_path=audio_path, duration_ms=duration_ms)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[extract_audio],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.UPLOAD_AUDIO)
|
||||
async def upload_audio(input: FilePipelineInput, ctx: Context) -> UploadAudioResult:
|
||||
"""Upload audio to S3/storage, return audio_url."""
|
||||
ctx.log(f"upload_audio: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
extract_result = ctx.task_output(extract_audio)
|
||||
audio_path = extract_result.audio_path
|
||||
|
||||
from reflector.storage import get_transcripts_storage # noqa: PLC0415
|
||||
|
||||
storage = get_transcripts_storage()
|
||||
if not storage:
|
||||
raise ValueError(
|
||||
"Storage backend required for file processing. "
|
||||
"Configure TRANSCRIPT_STORAGE_* settings."
|
||||
)
|
||||
|
||||
with open(audio_path, "rb") as f:
|
||||
audio_data = f.read()
|
||||
|
||||
storage_path = f"file_pipeline/{input.transcript_id}/audio.mp3"
|
||||
await storage.put_file(storage_path, audio_data)
|
||||
audio_url = await storage.get_file_url(storage_path)
|
||||
|
||||
ctx.log(f"upload_audio complete: {audio_url}")
|
||||
return UploadAudioResult(audio_url=audio_url, audio_path=audio_path)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[upload_audio],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_HEAVY),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=30,
|
||||
)
|
||||
@with_error_handling(TaskName.TRANSCRIBE)
|
||||
async def transcribe(input: FilePipelineInput, ctx: Context) -> TranscribeResult:
|
||||
"""Transcribe the audio file using the configured backend."""
|
||||
ctx.log(f"transcribe: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
upload_result = ctx.task_output(upload_audio)
|
||||
audio_url = upload_result.audio_url
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.transcripts import transcripts_controller # noqa: PLC0415
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if not transcript:
|
||||
raise ValueError(f"Transcript {input.transcript_id} not found")
|
||||
source_language = transcript.source_language
|
||||
|
||||
from reflector.pipelines.transcription_helpers import ( # noqa: PLC0415
|
||||
transcribe_file_with_processor,
|
||||
)
|
||||
|
||||
result = await transcribe_file_with_processor(audio_url, source_language)
|
||||
|
||||
ctx.log(f"transcribe complete: {len(result.words)} words")
|
||||
return TranscribeResult(
|
||||
words=[w.model_dump() for w in result.words],
|
||||
translation=result.translation,
|
||||
)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[upload_audio],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_HEAVY),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=30,
|
||||
)
|
||||
@with_error_handling(TaskName.DIARIZE)
|
||||
async def diarize(input: FilePipelineInput, ctx: Context) -> DiarizeResult:
|
||||
"""Diarize the audio file (speaker identification)."""
|
||||
ctx.log(f"diarize: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
if not settings.DIARIZATION_BACKEND:
|
||||
ctx.log("diarize: diarization disabled, skipping")
|
||||
return DiarizeResult(diarization=None)
|
||||
|
||||
upload_result = ctx.task_output(upload_audio)
|
||||
audio_url = upload_result.audio_url
|
||||
|
||||
from reflector.processors.file_diarization import ( # noqa: PLC0415
|
||||
FileDiarizationInput,
|
||||
)
|
||||
from reflector.processors.file_diarization_auto import ( # noqa: PLC0415
|
||||
FileDiarizationAutoProcessor,
|
||||
)
|
||||
|
||||
processor = FileDiarizationAutoProcessor()
|
||||
input_data = FileDiarizationInput(audio_url=audio_url)
|
||||
|
||||
result = None
|
||||
|
||||
async def capture_result(diarization_output):
|
||||
nonlocal result
|
||||
result = diarization_output.diarization
|
||||
|
||||
try:
|
||||
processor.on(capture_result)
|
||||
await processor.push(input_data)
|
||||
await processor.flush()
|
||||
except Exception as e:
|
||||
logger.error(f"Diarization failed: {e}")
|
||||
return DiarizeResult(diarization=None)
|
||||
|
||||
ctx.log(f"diarize complete: {len(result) if result else 0} segments")
|
||||
return DiarizeResult(diarization=list(result) if result else None)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[upload_audio],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.GENERATE_WAVEFORM)
|
||||
async def generate_waveform(input: FilePipelineInput, ctx: Context) -> WaveformResult:
|
||||
"""Generate audio waveform visualization."""
|
||||
ctx.log(f"generate_waveform: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
upload_result = ctx.task_output(upload_audio)
|
||||
audio_path = upload_result.audio_path
|
||||
|
||||
from reflector.db.transcripts import ( # noqa: PLC0415
|
||||
TranscriptWaveform,
|
||||
transcripts_controller,
|
||||
)
|
||||
|
||||
waveform = get_audio_waveform(
|
||||
path=Path(audio_path), segments_count=WAVEFORM_SEGMENTS
|
||||
)
|
||||
|
||||
async with fresh_db_connection():
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if transcript:
|
||||
transcript.data_path.mkdir(parents=True, exist_ok=True)
|
||||
with open(transcript.audio_waveform_filename, "w") as f:
|
||||
json.dump(waveform, f)
|
||||
|
||||
waveform_data = TranscriptWaveform(waveform=waveform)
|
||||
await append_event_and_broadcast(
|
||||
input.transcript_id,
|
||||
transcript,
|
||||
"WAVEFORM",
|
||||
waveform_data,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
ctx.log("generate_waveform complete")
|
||||
return WaveformResult(waveform_generated=True)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[transcribe, diarize, generate_waveform],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.ASSEMBLE_TRANSCRIPT)
|
||||
async def assemble_transcript(
|
||||
input: FilePipelineInput, ctx: Context
|
||||
) -> AssembleTranscriptResult:
|
||||
"""Merge transcription + diarization results."""
|
||||
ctx.log(f"assemble_transcript: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
transcribe_result = ctx.task_output(transcribe)
|
||||
diarize_result = ctx.task_output(diarize)
|
||||
|
||||
from reflector.processors.transcript_diarization_assembler import ( # noqa: PLC0415
|
||||
TranscriptDiarizationAssemblerInput,
|
||||
TranscriptDiarizationAssemblerProcessor,
|
||||
)
|
||||
from reflector.processors.types import ( # noqa: PLC0415
|
||||
DiarizationSegment,
|
||||
Word,
|
||||
)
|
||||
from reflector.processors.types import ( # noqa: PLC0415
|
||||
Transcript as TranscriptType,
|
||||
)
|
||||
|
||||
words = [Word(**w) for w in transcribe_result.words]
|
||||
transcript_data = TranscriptType(
|
||||
words=words, translation=transcribe_result.translation
|
||||
)
|
||||
|
||||
diarization = None
|
||||
if diarize_result.diarization:
|
||||
diarization = [DiarizationSegment(**s) for s in diarize_result.diarization]
|
||||
|
||||
processor = TranscriptDiarizationAssemblerProcessor()
|
||||
assembler_input = TranscriptDiarizationAssemblerInput(
|
||||
transcript=transcript_data, diarization=diarization or []
|
||||
)
|
||||
|
||||
diarized_transcript = None
|
||||
|
||||
async def capture_result(transcript):
|
||||
nonlocal diarized_transcript
|
||||
diarized_transcript = transcript
|
||||
|
||||
processor.on(capture_result)
|
||||
await processor.push(assembler_input)
|
||||
await processor.flush()
|
||||
|
||||
if not diarized_transcript:
|
||||
raise ValueError("No diarized transcript captured")
|
||||
|
||||
# Save the assembled transcript events to the database
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.transcripts import ( # noqa: PLC0415
|
||||
TranscriptText,
|
||||
transcripts_controller,
|
||||
)
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if transcript:
|
||||
assembled_text = diarized_transcript.text if diarized_transcript else ""
|
||||
assembled_translation = (
|
||||
diarized_transcript.translation if diarized_transcript else None
|
||||
)
|
||||
await append_event_and_broadcast(
|
||||
input.transcript_id,
|
||||
transcript,
|
||||
"TRANSCRIPT",
|
||||
TranscriptText(text=assembled_text, translation=assembled_translation),
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
ctx.log("assemble_transcript complete")
|
||||
return AssembleTranscriptResult(assembled=True)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[assemble_transcript],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_HEAVY),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=30,
|
||||
)
|
||||
@with_error_handling(TaskName.DETECT_TOPICS)
|
||||
async def detect_topics(input: FilePipelineInput, ctx: Context) -> TopicsResult:
|
||||
"""Detect topics from the assembled transcript."""
|
||||
ctx.log(f"detect_topics: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
# Re-read the transcript to get the diarized words
|
||||
transcribe_result = ctx.task_output(transcribe)
|
||||
diarize_result = ctx.task_output(diarize)
|
||||
|
||||
from reflector.db.transcripts import ( # noqa: PLC0415
|
||||
TranscriptTopic,
|
||||
transcripts_controller,
|
||||
)
|
||||
from reflector.processors.transcript_diarization_assembler import ( # noqa: PLC0415
|
||||
TranscriptDiarizationAssemblerInput,
|
||||
TranscriptDiarizationAssemblerProcessor,
|
||||
)
|
||||
from reflector.processors.types import ( # noqa: PLC0415
|
||||
DiarizationSegment,
|
||||
Word,
|
||||
)
|
||||
from reflector.processors.types import ( # noqa: PLC0415
|
||||
Transcript as TranscriptType,
|
||||
)
|
||||
|
||||
words = [Word(**w) for w in transcribe_result.words]
|
||||
transcript_data = TranscriptType(
|
||||
words=words, translation=transcribe_result.translation
|
||||
)
|
||||
|
||||
diarization = None
|
||||
if diarize_result.diarization:
|
||||
diarization = [DiarizationSegment(**s) for s in diarize_result.diarization]
|
||||
|
||||
# Re-assemble to get the diarized transcript for topic detection
|
||||
processor = TranscriptDiarizationAssemblerProcessor()
|
||||
assembler_input = TranscriptDiarizationAssemblerInput(
|
||||
transcript=transcript_data, diarization=diarization or []
|
||||
)
|
||||
|
||||
diarized_transcript = None
|
||||
|
||||
async def capture_result(transcript):
|
||||
nonlocal diarized_transcript
|
||||
diarized_transcript = transcript
|
||||
|
||||
processor.on(capture_result)
|
||||
await processor.push(assembler_input)
|
||||
await processor.flush()
|
||||
|
||||
if not diarized_transcript:
|
||||
raise ValueError("No diarized transcript for topic detection")
|
||||
|
||||
async with fresh_db_connection():
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if not transcript:
|
||||
raise ValueError(f"Transcript {input.transcript_id} not found")
|
||||
target_language = transcript.target_language
|
||||
|
||||
empty_pipeline = topic_processing.EmptyPipeline(logger=logger)
|
||||
|
||||
async def on_topic_callback(data):
|
||||
topic = TranscriptTopic(
|
||||
title=data.title,
|
||||
summary=data.summary,
|
||||
timestamp=data.timestamp,
|
||||
transcript=data.transcript.text
|
||||
if hasattr(data.transcript, "text")
|
||||
else "",
|
||||
words=data.transcript.words
|
||||
if hasattr(data.transcript, "words")
|
||||
else [],
|
||||
)
|
||||
await transcripts_controller.upsert_topic(transcript, topic)
|
||||
await append_event_and_broadcast(
|
||||
input.transcript_id, transcript, "TOPIC", topic, logger=logger
|
||||
)
|
||||
|
||||
topics = await topic_processing.detect_topics(
|
||||
diarized_transcript,
|
||||
target_language,
|
||||
on_topic_callback=on_topic_callback,
|
||||
empty_pipeline=empty_pipeline,
|
||||
)
|
||||
|
||||
ctx.log(f"detect_topics complete: {len(topics)} topics")
|
||||
return TopicsResult(topics=topics)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[detect_topics],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_TITLE),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=15,
|
||||
)
|
||||
@with_error_handling(TaskName.GENERATE_TITLE)
|
||||
async def generate_title(input: FilePipelineInput, ctx: Context) -> TitleResult:
|
||||
"""Generate meeting title using LLM."""
|
||||
ctx.log(f"generate_title: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
topics_result = ctx.task_output(detect_topics)
|
||||
topics = topics_result.topics
|
||||
|
||||
from reflector.db.transcripts import ( # noqa: PLC0415
|
||||
TranscriptFinalTitle,
|
||||
transcripts_controller,
|
||||
)
|
||||
|
||||
empty_pipeline = topic_processing.EmptyPipeline(logger=logger)
|
||||
title_result = None
|
||||
|
||||
async with fresh_db_connection():
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if not transcript:
|
||||
raise ValueError(f"Transcript {input.transcript_id} not found")
|
||||
|
||||
async def on_title_callback(data):
|
||||
nonlocal title_result
|
||||
title_result = data.title
|
||||
final_title = TranscriptFinalTitle(title=data.title)
|
||||
if not transcript.title:
|
||||
await transcripts_controller.update(
|
||||
transcript, {"title": final_title.title}
|
||||
)
|
||||
await append_event_and_broadcast(
|
||||
input.transcript_id,
|
||||
transcript,
|
||||
"FINAL_TITLE",
|
||||
final_title,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
await topic_processing.generate_title(
|
||||
topics,
|
||||
on_title_callback=on_title_callback,
|
||||
empty_pipeline=empty_pipeline,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
ctx.log(f"generate_title complete: '{title_result}'")
|
||||
return TitleResult(title=title_result)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[detect_topics],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_HEAVY),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=30,
|
||||
)
|
||||
@with_error_handling(TaskName.GENERATE_SUMMARIES)
|
||||
async def generate_summaries(input: FilePipelineInput, ctx: Context) -> SummariesResult:
|
||||
"""Generate long/short summaries and action items."""
|
||||
ctx.log(f"generate_summaries: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
topics_result = ctx.task_output(detect_topics)
|
||||
topics = topics_result.topics
|
||||
|
||||
from reflector.db.transcripts import ( # noqa: PLC0415
|
||||
TranscriptActionItems,
|
||||
TranscriptFinalLongSummary,
|
||||
TranscriptFinalShortSummary,
|
||||
transcripts_controller,
|
||||
)
|
||||
|
||||
empty_pipeline = topic_processing.EmptyPipeline(logger=logger)
|
||||
|
||||
async with fresh_db_connection():
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if not transcript:
|
||||
raise ValueError(f"Transcript {input.transcript_id} not found")
|
||||
|
||||
async def on_long_summary_callback(data):
|
||||
final_long = TranscriptFinalLongSummary(long_summary=data.long_summary)
|
||||
await transcripts_controller.update(
|
||||
transcript, {"long_summary": final_long.long_summary}
|
||||
)
|
||||
await append_event_and_broadcast(
|
||||
input.transcript_id,
|
||||
transcript,
|
||||
"FINAL_LONG_SUMMARY",
|
||||
final_long,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
async def on_short_summary_callback(data):
|
||||
final_short = TranscriptFinalShortSummary(short_summary=data.short_summary)
|
||||
await transcripts_controller.update(
|
||||
transcript, {"short_summary": final_short.short_summary}
|
||||
)
|
||||
await append_event_and_broadcast(
|
||||
input.transcript_id,
|
||||
transcript,
|
||||
"FINAL_SHORT_SUMMARY",
|
||||
final_short,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
async def on_action_items_callback(data):
|
||||
action_items = TranscriptActionItems(action_items=data.action_items)
|
||||
await transcripts_controller.update(
|
||||
transcript, {"action_items": action_items.action_items}
|
||||
)
|
||||
await append_event_and_broadcast(
|
||||
input.transcript_id,
|
||||
transcript,
|
||||
"ACTION_ITEMS",
|
||||
action_items,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
await topic_processing.generate_summaries(
|
||||
topics,
|
||||
transcript,
|
||||
on_long_summary_callback=on_long_summary_callback,
|
||||
on_short_summary_callback=on_short_summary_callback,
|
||||
on_action_items_callback=on_action_items_callback,
|
||||
empty_pipeline=empty_pipeline,
|
||||
logger=logger,
|
||||
)
|
||||
|
||||
ctx.log("generate_summaries complete")
|
||||
return SummariesResult(generated=True)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[generate_title, generate_summaries],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_SHORT),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=5,
|
||||
)
|
||||
@with_error_handling(TaskName.FINALIZE)
|
||||
async def finalize(input: FilePipelineInput, ctx: Context) -> FinalizeResult:
|
||||
"""Set transcript status to 'ended' and broadcast."""
|
||||
ctx.log("finalize: setting status to 'ended'")
|
||||
|
||||
async with fresh_db_connection():
|
||||
await set_status_and_broadcast(input.transcript_id, "ended", logger=logger)
|
||||
|
||||
ctx.log("finalize complete")
|
||||
return FinalizeResult(status="COMPLETED")
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[finalize],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_SHORT),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.CLEANUP_CONSENT, set_error_status=False)
|
||||
async def cleanup_consent(input: FilePipelineInput, ctx: Context) -> ConsentResult:
|
||||
"""Check consent and delete audio files if any participant denied."""
|
||||
ctx.log(f"cleanup_consent: transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.meetings import ( # noqa: PLC0415
|
||||
meeting_consent_controller,
|
||||
meetings_controller,
|
||||
)
|
||||
from reflector.db.recordings import recordings_controller # noqa: PLC0415
|
||||
from reflector.db.transcripts import transcripts_controller # noqa: PLC0415
|
||||
from reflector.storage import get_transcripts_storage # noqa: PLC0415
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if not transcript:
|
||||
ctx.log("cleanup_consent: transcript not found")
|
||||
return ConsentResult()
|
||||
|
||||
consent_denied = False
|
||||
recording = None
|
||||
if transcript.recording_id:
|
||||
recording = await recordings_controller.get_by_id(transcript.recording_id)
|
||||
if recording and recording.meeting_id:
|
||||
meeting = await meetings_controller.get_by_id(recording.meeting_id)
|
||||
if meeting:
|
||||
consent_denied = await meeting_consent_controller.has_any_denial(
|
||||
meeting.id
|
||||
)
|
||||
|
||||
if not consent_denied:
|
||||
ctx.log("cleanup_consent: consent approved, keeping all files")
|
||||
return ConsentResult()
|
||||
|
||||
ctx.log("cleanup_consent: consent denied, deleting audio files")
|
||||
|
||||
deletion_errors = []
|
||||
if recording and recording.bucket_name:
|
||||
keys_to_delete = []
|
||||
if recording.track_keys:
|
||||
keys_to_delete = recording.track_keys
|
||||
elif recording.object_key:
|
||||
keys_to_delete = [recording.object_key]
|
||||
|
||||
master_storage = get_transcripts_storage()
|
||||
for key in keys_to_delete:
|
||||
try:
|
||||
await master_storage.delete_file(key, bucket=recording.bucket_name)
|
||||
ctx.log(f"Deleted recording file: {recording.bucket_name}/{key}")
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to delete {key}: {e}"
|
||||
logger.error(error_msg, exc_info=True)
|
||||
deletion_errors.append(error_msg)
|
||||
|
||||
if transcript.audio_location == "storage":
|
||||
storage = get_transcripts_storage()
|
||||
try:
|
||||
await storage.delete_file(transcript.storage_audio_path)
|
||||
ctx.log(f"Deleted processed audio: {transcript.storage_audio_path}")
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to delete processed audio: {e}"
|
||||
logger.error(error_msg, exc_info=True)
|
||||
deletion_errors.append(error_msg)
|
||||
|
||||
try:
|
||||
if (
|
||||
hasattr(transcript, "audio_mp3_filename")
|
||||
and transcript.audio_mp3_filename
|
||||
):
|
||||
transcript.audio_mp3_filename.unlink(missing_ok=True)
|
||||
if (
|
||||
hasattr(transcript, "audio_wav_filename")
|
||||
and transcript.audio_wav_filename
|
||||
):
|
||||
transcript.audio_wav_filename.unlink(missing_ok=True)
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to delete local audio files: {e}"
|
||||
logger.error(error_msg, exc_info=True)
|
||||
deletion_errors.append(error_msg)
|
||||
|
||||
if deletion_errors:
|
||||
logger.warning(
|
||||
"[Hatchet] cleanup_consent completed with errors",
|
||||
transcript_id=input.transcript_id,
|
||||
error_count=len(deletion_errors),
|
||||
)
|
||||
else:
|
||||
await transcripts_controller.update(transcript, {"audio_deleted": True})
|
||||
ctx.log("cleanup_consent: all audio deleted successfully")
|
||||
|
||||
return ConsentResult()
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[cleanup_consent],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_SHORT),
|
||||
retries=5,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=15,
|
||||
)
|
||||
@with_error_handling(TaskName.POST_ZULIP, set_error_status=False)
|
||||
async def post_zulip(input: FilePipelineInput, ctx: Context) -> ZulipResult:
|
||||
"""Post notification to Zulip."""
|
||||
ctx.log(f"post_zulip: transcript_id={input.transcript_id}")
|
||||
|
||||
if not settings.ZULIP_REALM:
|
||||
ctx.log("post_zulip skipped (Zulip not configured)")
|
||||
return ZulipResult(zulip_message_id=None, skipped=True)
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.transcripts import transcripts_controller # noqa: PLC0415
|
||||
from reflector.zulip import post_transcript_notification # noqa: PLC0415
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if transcript:
|
||||
message_id = await post_transcript_notification(transcript)
|
||||
ctx.log(f"post_zulip complete: zulip_message_id={message_id}")
|
||||
else:
|
||||
message_id = None
|
||||
|
||||
return ZulipResult(zulip_message_id=message_id)
|
||||
|
||||
|
||||
@file_pipeline.task(
|
||||
parents=[cleanup_consent],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=5,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=15,
|
||||
)
|
||||
@with_error_handling(TaskName.SEND_WEBHOOK, set_error_status=False)
|
||||
async def send_webhook(input: FilePipelineInput, ctx: Context) -> WebhookResult:
|
||||
"""Send completion webhook to external service."""
|
||||
ctx.log(f"send_webhook: transcript_id={input.transcript_id}")
|
||||
|
||||
if not input.room_id:
|
||||
ctx.log("send_webhook skipped (no room_id)")
|
||||
return WebhookResult(webhook_sent=False, skipped=True)
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.rooms import rooms_controller # noqa: PLC0415
|
||||
from reflector.utils.webhook import ( # noqa: PLC0415
|
||||
fetch_transcript_webhook_payload,
|
||||
send_webhook_request,
|
||||
)
|
||||
|
||||
room = await rooms_controller.get_by_id(input.room_id)
|
||||
if not room or not room.webhook_url:
|
||||
ctx.log("send_webhook skipped (no webhook_url configured)")
|
||||
return WebhookResult(webhook_sent=False, skipped=True)
|
||||
|
||||
payload = await fetch_transcript_webhook_payload(
|
||||
transcript_id=input.transcript_id,
|
||||
room_id=input.room_id,
|
||||
)
|
||||
|
||||
if isinstance(payload, str):
|
||||
ctx.log(f"send_webhook skipped (could not build payload): {payload}")
|
||||
return WebhookResult(webhook_sent=False, skipped=True)
|
||||
|
||||
import httpx # noqa: PLC0415
|
||||
|
||||
try:
|
||||
response = await send_webhook_request(
|
||||
url=room.webhook_url,
|
||||
payload=payload,
|
||||
event_type="transcript.completed",
|
||||
webhook_secret=room.webhook_secret,
|
||||
timeout=30.0,
|
||||
)
|
||||
ctx.log(f"send_webhook complete: status_code={response.status_code}")
|
||||
return WebhookResult(webhook_sent=True, response_code=response.status_code)
|
||||
except httpx.HTTPStatusError as e:
|
||||
ctx.log(f"send_webhook failed (HTTP {e.response.status_code}), continuing")
|
||||
return WebhookResult(
|
||||
webhook_sent=False, response_code=e.response.status_code
|
||||
)
|
||||
except (httpx.ConnectError, httpx.TimeoutException) as e:
|
||||
ctx.log(f"send_webhook failed ({e}), continuing")
|
||||
return WebhookResult(webhook_sent=False)
|
||||
except Exception as e:
|
||||
ctx.log(f"send_webhook unexpected error: {e}")
|
||||
return WebhookResult(webhook_sent=False)
|
||||
|
||||
|
||||
# --- On failure handler ---
|
||||
|
||||
|
||||
async def on_workflow_failure(input: FilePipelineInput, ctx: Context) -> None:
|
||||
"""Set transcript status to 'error' only if not already 'ended'."""
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.transcripts import transcripts_controller # noqa: PLC0415
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if transcript and transcript.status == "ended":
|
||||
logger.info(
|
||||
"[Hatchet] FilePipeline on_workflow_failure: transcript already ended, skipping error status",
|
||||
transcript_id=input.transcript_id,
|
||||
)
|
||||
ctx.log(
|
||||
"on_workflow_failure: transcript already ended, skipping error status"
|
||||
)
|
||||
return
|
||||
await set_workflow_error_status(input.transcript_id)
|
||||
|
||||
|
||||
@file_pipeline.on_failure_task()
|
||||
async def _register_on_workflow_failure(input: FilePipelineInput, ctx: Context) -> None:
|
||||
await on_workflow_failure(input, ctx)
|
||||
389
server/reflector/hatchet/workflows/live_post_pipeline.py
Normal file
389
server/reflector/hatchet/workflows/live_post_pipeline.py
Normal file
@@ -0,0 +1,389 @@
|
||||
"""
|
||||
Hatchet workflow: LivePostProcessingPipeline
|
||||
|
||||
Post-processing pipeline for live WebRTC meetings.
|
||||
Triggered after a live meeting ends. Orchestrates:
|
||||
Left branch: waveform → convert_mp3 → upload_mp3 → remove_upload → diarize → cleanup_consent
|
||||
Right branch: generate_title (parallel with left branch)
|
||||
Fan-in: final_summaries → post_zulip → send_webhook
|
||||
|
||||
Note: This file uses deferred imports (inside functions/tasks) intentionally.
|
||||
Hatchet workers run in forked processes; fresh imports per task ensure DB connections
|
||||
are not shared across forks, avoiding connection pooling issues.
|
||||
"""
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
from hatchet_sdk import Context
|
||||
from pydantic import BaseModel
|
||||
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
from reflector.hatchet.constants import (
|
||||
TIMEOUT_HEAVY,
|
||||
TIMEOUT_MEDIUM,
|
||||
TIMEOUT_SHORT,
|
||||
TIMEOUT_TITLE,
|
||||
TaskName,
|
||||
)
|
||||
from reflector.hatchet.workflows.daily_multitrack_pipeline import (
|
||||
fresh_db_connection,
|
||||
set_workflow_error_status,
|
||||
with_error_handling,
|
||||
)
|
||||
from reflector.hatchet.workflows.models import (
|
||||
ConsentResult,
|
||||
TitleResult,
|
||||
WaveformResult,
|
||||
WebhookResult,
|
||||
ZulipResult,
|
||||
)
|
||||
from reflector.logger import logger
|
||||
from reflector.settings import settings
|
||||
|
||||
|
||||
class LivePostPipelineInput(BaseModel):
|
||||
transcript_id: str
|
||||
room_id: str | None = None
|
||||
|
||||
|
||||
# --- Result models specific to live post pipeline ---
|
||||
|
||||
|
||||
class ConvertMp3Result(BaseModel):
|
||||
converted: bool
|
||||
|
||||
|
||||
class UploadMp3Result(BaseModel):
|
||||
uploaded: bool
|
||||
|
||||
|
||||
class RemoveUploadResult(BaseModel):
|
||||
removed: bool
|
||||
|
||||
|
||||
class DiarizeResult(BaseModel):
|
||||
diarized: bool
|
||||
|
||||
|
||||
class FinalSummariesResult(BaseModel):
|
||||
generated: bool
|
||||
|
||||
|
||||
hatchet = HatchetClientManager.get_client()
|
||||
|
||||
live_post_pipeline = hatchet.workflow(
|
||||
name="LivePostProcessingPipeline", input_validator=LivePostPipelineInput
|
||||
)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.WAVEFORM)
|
||||
async def waveform(input: LivePostPipelineInput, ctx: Context) -> WaveformResult:
|
||||
"""Generate waveform visualization from recorded audio."""
|
||||
ctx.log(f"waveform: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.transcripts import transcripts_controller # noqa: PLC0415
|
||||
from reflector.pipelines.main_live_pipeline import ( # noqa: PLC0415
|
||||
PipelineMainWaveform,
|
||||
)
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if not transcript:
|
||||
raise ValueError(f"Transcript {input.transcript_id} not found")
|
||||
|
||||
runner = PipelineMainWaveform(transcript_id=transcript.id)
|
||||
await runner.run()
|
||||
|
||||
ctx.log("waveform complete")
|
||||
return WaveformResult(waveform_generated=True)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_TITLE),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=15,
|
||||
)
|
||||
@with_error_handling(TaskName.GENERATE_TITLE)
|
||||
async def generate_title(input: LivePostPipelineInput, ctx: Context) -> TitleResult:
|
||||
"""Generate meeting title from topics (runs in parallel with audio chain)."""
|
||||
ctx.log(f"generate_title: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.pipelines.main_live_pipeline import ( # noqa: PLC0415
|
||||
PipelineMainTitle,
|
||||
)
|
||||
|
||||
runner = PipelineMainTitle(transcript_id=input.transcript_id)
|
||||
await runner.run()
|
||||
|
||||
ctx.log("generate_title complete")
|
||||
return TitleResult(title=None)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
parents=[waveform],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.CONVERT_MP3)
|
||||
async def convert_mp3(input: LivePostPipelineInput, ctx: Context) -> ConvertMp3Result:
|
||||
"""Convert WAV recording to MP3."""
|
||||
ctx.log(f"convert_mp3: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.pipelines.main_live_pipeline import ( # noqa: PLC0415
|
||||
pipeline_convert_to_mp3,
|
||||
)
|
||||
|
||||
await pipeline_convert_to_mp3(transcript_id=input.transcript_id)
|
||||
|
||||
ctx.log("convert_mp3 complete")
|
||||
return ConvertMp3Result(converted=True)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
parents=[convert_mp3],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.UPLOAD_MP3)
|
||||
async def upload_mp3(input: LivePostPipelineInput, ctx: Context) -> UploadMp3Result:
|
||||
"""Upload MP3 to external storage."""
|
||||
ctx.log(f"upload_mp3: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.pipelines.main_live_pipeline import ( # noqa: PLC0415
|
||||
pipeline_upload_mp3,
|
||||
)
|
||||
|
||||
await pipeline_upload_mp3(transcript_id=input.transcript_id)
|
||||
|
||||
ctx.log("upload_mp3 complete")
|
||||
return UploadMp3Result(uploaded=True)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
parents=[upload_mp3],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_SHORT),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=5,
|
||||
)
|
||||
@with_error_handling(TaskName.REMOVE_UPLOAD)
|
||||
async def remove_upload(
|
||||
input: LivePostPipelineInput, ctx: Context
|
||||
) -> RemoveUploadResult:
|
||||
"""Remove the original upload file."""
|
||||
ctx.log(f"remove_upload: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.pipelines.main_live_pipeline import ( # noqa: PLC0415
|
||||
pipeline_remove_upload,
|
||||
)
|
||||
|
||||
await pipeline_remove_upload(transcript_id=input.transcript_id)
|
||||
|
||||
ctx.log("remove_upload complete")
|
||||
return RemoveUploadResult(removed=True)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
parents=[remove_upload],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_HEAVY),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=30,
|
||||
)
|
||||
@with_error_handling(TaskName.DIARIZE)
|
||||
async def diarize(input: LivePostPipelineInput, ctx: Context) -> DiarizeResult:
|
||||
"""Run diarization on the recorded audio."""
|
||||
ctx.log(f"diarize: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.pipelines.main_live_pipeline import ( # noqa: PLC0415
|
||||
pipeline_diarization,
|
||||
)
|
||||
|
||||
await pipeline_diarization(transcript_id=input.transcript_id)
|
||||
|
||||
ctx.log("diarize complete")
|
||||
return DiarizeResult(diarized=True)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
parents=[diarize],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_SHORT),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=10,
|
||||
)
|
||||
@with_error_handling(TaskName.CLEANUP_CONSENT, set_error_status=False)
|
||||
async def cleanup_consent(input: LivePostPipelineInput, ctx: Context) -> ConsentResult:
|
||||
"""Check consent and delete audio files if any participant denied."""
|
||||
ctx.log(f"cleanup_consent: transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.pipelines.main_live_pipeline import ( # noqa: PLC0415
|
||||
cleanup_consent as _cleanup_consent,
|
||||
)
|
||||
|
||||
await _cleanup_consent(transcript_id=input.transcript_id)
|
||||
|
||||
ctx.log("cleanup_consent complete")
|
||||
return ConsentResult()
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
parents=[cleanup_consent, generate_title],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_HEAVY),
|
||||
retries=3,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=30,
|
||||
)
|
||||
@with_error_handling(TaskName.FINAL_SUMMARIES)
|
||||
async def final_summaries(
|
||||
input: LivePostPipelineInput, ctx: Context
|
||||
) -> FinalSummariesResult:
|
||||
"""Generate final summaries (fan-in after audio chain + title)."""
|
||||
ctx.log(f"final_summaries: starting for transcript_id={input.transcript_id}")
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.pipelines.main_live_pipeline import ( # noqa: PLC0415
|
||||
pipeline_summaries,
|
||||
)
|
||||
|
||||
await pipeline_summaries(transcript_id=input.transcript_id)
|
||||
|
||||
ctx.log("final_summaries complete")
|
||||
return FinalSummariesResult(generated=True)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
parents=[final_summaries],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_SHORT),
|
||||
retries=5,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=15,
|
||||
)
|
||||
@with_error_handling(TaskName.POST_ZULIP, set_error_status=False)
|
||||
async def post_zulip(input: LivePostPipelineInput, ctx: Context) -> ZulipResult:
|
||||
"""Post notification to Zulip."""
|
||||
ctx.log(f"post_zulip: transcript_id={input.transcript_id}")
|
||||
|
||||
if not settings.ZULIP_REALM:
|
||||
ctx.log("post_zulip skipped (Zulip not configured)")
|
||||
return ZulipResult(zulip_message_id=None, skipped=True)
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.transcripts import transcripts_controller # noqa: PLC0415
|
||||
from reflector.zulip import post_transcript_notification # noqa: PLC0415
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if transcript:
|
||||
message_id = await post_transcript_notification(transcript)
|
||||
ctx.log(f"post_zulip complete: zulip_message_id={message_id}")
|
||||
else:
|
||||
message_id = None
|
||||
|
||||
return ZulipResult(zulip_message_id=message_id)
|
||||
|
||||
|
||||
@live_post_pipeline.task(
|
||||
parents=[final_summaries],
|
||||
execution_timeout=timedelta(seconds=TIMEOUT_MEDIUM),
|
||||
retries=5,
|
||||
backoff_factor=2.0,
|
||||
backoff_max_seconds=15,
|
||||
)
|
||||
@with_error_handling(TaskName.SEND_WEBHOOK, set_error_status=False)
|
||||
async def send_webhook(input: LivePostPipelineInput, ctx: Context) -> WebhookResult:
|
||||
"""Send completion webhook to external service."""
|
||||
ctx.log(f"send_webhook: transcript_id={input.transcript_id}")
|
||||
|
||||
if not input.room_id:
|
||||
ctx.log("send_webhook skipped (no room_id)")
|
||||
return WebhookResult(webhook_sent=False, skipped=True)
|
||||
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.rooms import rooms_controller # noqa: PLC0415
|
||||
from reflector.utils.webhook import ( # noqa: PLC0415
|
||||
fetch_transcript_webhook_payload,
|
||||
send_webhook_request,
|
||||
)
|
||||
|
||||
room = await rooms_controller.get_by_id(input.room_id)
|
||||
if not room or not room.webhook_url:
|
||||
ctx.log("send_webhook skipped (no webhook_url configured)")
|
||||
return WebhookResult(webhook_sent=False, skipped=True)
|
||||
|
||||
payload = await fetch_transcript_webhook_payload(
|
||||
transcript_id=input.transcript_id,
|
||||
room_id=input.room_id,
|
||||
)
|
||||
|
||||
if isinstance(payload, str):
|
||||
ctx.log(f"send_webhook skipped (could not build payload): {payload}")
|
||||
return WebhookResult(webhook_sent=False, skipped=True)
|
||||
|
||||
import httpx # noqa: PLC0415
|
||||
|
||||
try:
|
||||
response = await send_webhook_request(
|
||||
url=room.webhook_url,
|
||||
payload=payload,
|
||||
event_type="transcript.completed",
|
||||
webhook_secret=room.webhook_secret,
|
||||
timeout=30.0,
|
||||
)
|
||||
ctx.log(f"send_webhook complete: status_code={response.status_code}")
|
||||
return WebhookResult(webhook_sent=True, response_code=response.status_code)
|
||||
except httpx.HTTPStatusError as e:
|
||||
ctx.log(f"send_webhook failed (HTTP {e.response.status_code}), continuing")
|
||||
return WebhookResult(
|
||||
webhook_sent=False, response_code=e.response.status_code
|
||||
)
|
||||
except (httpx.ConnectError, httpx.TimeoutException) as e:
|
||||
ctx.log(f"send_webhook failed ({e}), continuing")
|
||||
return WebhookResult(webhook_sent=False)
|
||||
except Exception as e:
|
||||
ctx.log(f"send_webhook unexpected error: {e}")
|
||||
return WebhookResult(webhook_sent=False)
|
||||
|
||||
|
||||
# --- On failure handler ---
|
||||
|
||||
|
||||
async def on_workflow_failure(input: LivePostPipelineInput, ctx: Context) -> None:
|
||||
"""Set transcript status to 'error' only if not already 'ended'."""
|
||||
async with fresh_db_connection():
|
||||
from reflector.db.transcripts import transcripts_controller # noqa: PLC0415
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(input.transcript_id)
|
||||
if transcript and transcript.status == "ended":
|
||||
logger.info(
|
||||
"[Hatchet] LivePostProcessingPipeline on_workflow_failure: transcript already ended",
|
||||
transcript_id=input.transcript_id,
|
||||
)
|
||||
ctx.log(
|
||||
"on_workflow_failure: transcript already ended, skipping error status"
|
||||
)
|
||||
return
|
||||
await set_workflow_error_status(input.transcript_id)
|
||||
|
||||
|
||||
@live_post_pipeline.on_failure_task()
|
||||
async def _register_on_workflow_failure(
|
||||
input: LivePostPipelineInput, ctx: Context
|
||||
) -> None:
|
||||
await on_workflow_failure(input, ctx)
|
||||
@@ -17,7 +17,7 @@ from contextlib import asynccontextmanager
|
||||
from typing import Generic
|
||||
|
||||
import av
|
||||
from celery import chord, current_task, group, shared_task
|
||||
from celery import current_task, shared_task
|
||||
from pydantic import BaseModel
|
||||
from structlog import BoundLogger as Logger
|
||||
|
||||
@@ -397,7 +397,9 @@ class PipelineMainLive(PipelineMainBase):
|
||||
# when the pipeline ends, connect to the post pipeline
|
||||
logger.info("Pipeline main live ended", transcript_id=self.transcript_id)
|
||||
logger.info("Scheduling pipeline main post", transcript_id=self.transcript_id)
|
||||
pipeline_post(transcript_id=self.transcript_id)
|
||||
transcript = await transcripts_controller.get_by_id(self.transcript_id)
|
||||
room_id = transcript.room_id if transcript else None
|
||||
await pipeline_post(transcript_id=self.transcript_id, room_id=room_id)
|
||||
|
||||
|
||||
class PipelineMainDiarization(PipelineMainBase[AudioDiarizationInput]):
|
||||
@@ -792,29 +794,20 @@ async def task_pipeline_post_to_zulip(*, transcript_id: str):
|
||||
await pipeline_post_to_zulip(transcript_id=transcript_id)
|
||||
|
||||
|
||||
def pipeline_post(*, transcript_id: str):
|
||||
async def pipeline_post(*, transcript_id: str, room_id: str | None = None):
|
||||
"""
|
||||
Run the post pipeline
|
||||
Run the post pipeline via Hatchet.
|
||||
"""
|
||||
chain_mp3_and_diarize = (
|
||||
task_pipeline_waveform.si(transcript_id=transcript_id)
|
||||
| task_pipeline_convert_to_mp3.si(transcript_id=transcript_id)
|
||||
| task_pipeline_upload_mp3.si(transcript_id=transcript_id)
|
||||
| task_pipeline_remove_upload.si(transcript_id=transcript_id)
|
||||
| task_pipeline_diarization.si(transcript_id=transcript_id)
|
||||
| task_cleanup_consent.si(transcript_id=transcript_id)
|
||||
)
|
||||
chain_title_preview = task_pipeline_title.si(transcript_id=transcript_id)
|
||||
chain_final_summaries = task_pipeline_final_summaries.si(
|
||||
transcript_id=transcript_id
|
||||
)
|
||||
from reflector.hatchet.client import HatchetClientManager # noqa: PLC0415
|
||||
|
||||
chain = chord(
|
||||
group(chain_mp3_and_diarize, chain_title_preview),
|
||||
chain_final_summaries,
|
||||
) | task_pipeline_post_to_zulip.si(transcript_id=transcript_id)
|
||||
|
||||
return chain.delay()
|
||||
await HatchetClientManager.start_workflow(
|
||||
"LivePostProcessingPipeline",
|
||||
{
|
||||
"transcript_id": str(transcript_id),
|
||||
"room_id": str(room_id) if room_id else None,
|
||||
},
|
||||
additional_metadata={"transcript_id": str(transcript_id)},
|
||||
)
|
||||
|
||||
|
||||
@get_transcript
|
||||
|
||||
@@ -10,7 +10,6 @@ from dataclasses import dataclass
|
||||
from typing import Literal, Union, assert_never
|
||||
|
||||
import celery
|
||||
from celery.result import AsyncResult
|
||||
from hatchet_sdk.clients.rest.exceptions import ApiException, NotFoundException
|
||||
from hatchet_sdk.clients.rest.models import V1TaskStatus
|
||||
|
||||
@@ -18,7 +17,6 @@ from reflector.db.recordings import recordings_controller
|
||||
from reflector.db.transcripts import Transcript, transcripts_controller
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
from reflector.logger import logger
|
||||
from reflector.pipelines.main_file_pipeline import task_pipeline_file_process
|
||||
from reflector.utils.string import NonEmptyString
|
||||
|
||||
|
||||
@@ -105,11 +103,8 @@ async def validate_transcript_for_processing(
|
||||
):
|
||||
return ValidationNotReady(detail="Recording is not ready for processing")
|
||||
|
||||
# Check Celery tasks
|
||||
# Check Celery tasks (multitrack still uses Celery for some paths)
|
||||
if task_is_scheduled_or_active(
|
||||
"reflector.pipelines.main_file_pipeline.task_pipeline_file_process",
|
||||
transcript_id=transcript.id,
|
||||
) or task_is_scheduled_or_active(
|
||||
"reflector.pipelines.main_multitrack_pipeline.task_pipeline_multitrack_process",
|
||||
transcript_id=transcript.id,
|
||||
):
|
||||
@@ -175,11 +170,8 @@ async def prepare_transcript_processing(validation: ValidationOk) -> PrepareResu
|
||||
|
||||
async def dispatch_transcript_processing(
|
||||
config: ProcessingConfig, force: bool = False
|
||||
) -> AsyncResult | None:
|
||||
"""Dispatch transcript processing to appropriate backend (Hatchet or Celery).
|
||||
|
||||
Returns AsyncResult for Celery tasks, None for Hatchet workflows.
|
||||
"""
|
||||
) -> None:
|
||||
"""Dispatch transcript processing to Hatchet workflow engine."""
|
||||
if isinstance(config, MultitrackProcessingConfig):
|
||||
# Multitrack processing always uses Hatchet (no Celery fallback)
|
||||
# First check if we can replay (outside transaction since it's read-only)
|
||||
@@ -275,7 +267,21 @@ async def dispatch_transcript_processing(
|
||||
return None
|
||||
|
||||
elif isinstance(config, FileProcessingConfig):
|
||||
return task_pipeline_file_process.delay(transcript_id=config.transcript_id)
|
||||
# File processing uses Hatchet workflow
|
||||
workflow_id = await HatchetClientManager.start_workflow(
|
||||
workflow_name="FilePipeline",
|
||||
input_data={"transcript_id": config.transcript_id},
|
||||
additional_metadata={"transcript_id": config.transcript_id},
|
||||
)
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(config.transcript_id)
|
||||
if transcript:
|
||||
await transcripts_controller.update(
|
||||
transcript, {"workflow_run_id": workflow_id}
|
||||
)
|
||||
|
||||
logger.info("File pipeline dispatched via Hatchet", workflow_id=workflow_id)
|
||||
return None
|
||||
else:
|
||||
assert_never(config)
|
||||
|
||||
|
||||
@@ -180,6 +180,7 @@ class Settings(BaseSettings):
|
||||
)
|
||||
|
||||
# Daily.co integration
|
||||
DAILY_API_URL: str = "https://api.daily.co/v1"
|
||||
DAILY_API_KEY: str | None = None
|
||||
DAILY_WEBHOOK_SECRET: str | None = None
|
||||
DAILY_SUBDOMAIN: str | None = None
|
||||
|
||||
@@ -7,7 +7,6 @@ import asyncio
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Tuple
|
||||
from urllib.parse import unquote, urlparse
|
||||
@@ -15,10 +14,8 @@ from urllib.parse import unquote, urlparse
|
||||
from botocore.exceptions import BotoCoreError, ClientError, NoCredentialsError
|
||||
|
||||
from reflector.db.transcripts import SourceKind, TranscriptTopic, transcripts_controller
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
from reflector.logger import logger
|
||||
from reflector.pipelines.main_file_pipeline import (
|
||||
task_pipeline_file_process as task_pipeline_file_process,
|
||||
)
|
||||
from reflector.pipelines.main_live_pipeline import pipeline_post as live_pipeline_post
|
||||
from reflector.pipelines.main_live_pipeline import (
|
||||
pipeline_process as live_pipeline_process,
|
||||
@@ -237,29 +234,22 @@ async def process_live_pipeline(
|
||||
# assert documented behaviour: after process, the pipeline isn't ended. this is the reason of calling pipeline_post
|
||||
assert pre_final_transcript.status != "ended"
|
||||
|
||||
# at this point, diarization is running but we have no access to it. run diarization in parallel - one will hopefully win after polling
|
||||
result = live_pipeline_post(transcript_id=transcript_id)
|
||||
|
||||
# result.ready() blocks even without await; it mutates result also
|
||||
while not result.ready():
|
||||
print(f"Status: {result.state}")
|
||||
time.sleep(2)
|
||||
# Trigger post-processing via Hatchet (fire-and-forget)
|
||||
await live_pipeline_post(transcript_id=transcript_id)
|
||||
print("Live post-processing pipeline triggered via Hatchet", file=sys.stderr)
|
||||
|
||||
|
||||
async def process_file_pipeline(
|
||||
transcript_id: TranscriptId,
|
||||
):
|
||||
"""Process audio/video file using the optimized file pipeline"""
|
||||
"""Process audio/video file using the optimized file pipeline via Hatchet"""
|
||||
|
||||
# task_pipeline_file_process is a Celery task, need to use .delay() for async execution
|
||||
result = task_pipeline_file_process.delay(transcript_id=transcript_id)
|
||||
|
||||
# Wait for the Celery task to complete
|
||||
while not result.ready():
|
||||
print(f"File pipeline status: {result.state}", file=sys.stderr)
|
||||
time.sleep(2)
|
||||
|
||||
logger.info("File pipeline processing complete")
|
||||
await HatchetClientManager.start_workflow(
|
||||
"FilePipeline",
|
||||
{"transcript_id": str(transcript_id)},
|
||||
additional_metadata={"transcript_id": str(transcript_id)},
|
||||
)
|
||||
print("File pipeline triggered via Hatchet", file=sys.stderr)
|
||||
|
||||
|
||||
async def process(
|
||||
@@ -293,7 +283,16 @@ async def process(
|
||||
|
||||
await handler(transcript_id)
|
||||
|
||||
await extract_result_from_entry(transcript_id, output_path)
|
||||
if pipeline == "file":
|
||||
# File pipeline is async via Hatchet — results not available immediately.
|
||||
# Use reflector.tools.process_transcript with --sync for polling.
|
||||
print(
|
||||
f"File pipeline dispatched for transcript {transcript_id}. "
|
||||
f"Results will be available once the Hatchet workflow completes.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
await extract_result_from_entry(transcript_id, output_path)
|
||||
finally:
|
||||
await database.disconnect()
|
||||
|
||||
|
||||
@@ -11,10 +11,8 @@ Usage:
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
from celery.result import AsyncResult
|
||||
from hatchet_sdk.clients.rest.models import V1TaskStatus
|
||||
|
||||
import reflector._warnings_filter # noqa: F401 -- side effect: suppress pydantic validate_default warning
|
||||
@@ -39,7 +37,7 @@ async def process_transcript_inner(
|
||||
on_validation: Callable[[ValidationResult], None],
|
||||
on_preprocess: Callable[[PrepareResult], None],
|
||||
force: bool = False,
|
||||
) -> AsyncResult | None:
|
||||
) -> None:
|
||||
validation = await validate_transcript_for_processing(transcript)
|
||||
on_validation(validation)
|
||||
config = await prepare_transcript_processing(validation)
|
||||
@@ -87,56 +85,39 @@ async def process_transcript(
|
||||
elif isinstance(config, FileProcessingConfig):
|
||||
print(f"Dispatching file pipeline", file=sys.stderr)
|
||||
|
||||
result = await process_transcript_inner(
|
||||
await process_transcript_inner(
|
||||
transcript,
|
||||
on_validation=on_validation,
|
||||
on_preprocess=on_preprocess,
|
||||
force=force,
|
||||
)
|
||||
|
||||
if result is None:
|
||||
# Hatchet workflow dispatched
|
||||
if sync:
|
||||
# Re-fetch transcript to get workflow_run_id
|
||||
transcript = await transcripts_controller.get_by_id(transcript_id)
|
||||
if not transcript or not transcript.workflow_run_id:
|
||||
print("Error: workflow_run_id not found", file=sys.stderr)
|
||||
if sync:
|
||||
# Re-fetch transcript to get workflow_run_id
|
||||
transcript = await transcripts_controller.get_by_id(transcript_id)
|
||||
if not transcript or not transcript.workflow_run_id:
|
||||
print("Error: workflow_run_id not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("Waiting for Hatchet workflow...", file=sys.stderr)
|
||||
while True:
|
||||
status = await HatchetClientManager.get_workflow_run_status(
|
||||
transcript.workflow_run_id
|
||||
)
|
||||
print(f" Status: {status.value}", file=sys.stderr)
|
||||
|
||||
if status == V1TaskStatus.COMPLETED:
|
||||
print("Workflow completed successfully", file=sys.stderr)
|
||||
break
|
||||
elif status in (V1TaskStatus.FAILED, V1TaskStatus.CANCELLED):
|
||||
print(f"Workflow failed: {status}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print("Waiting for Hatchet workflow...", file=sys.stderr)
|
||||
while True:
|
||||
status = await HatchetClientManager.get_workflow_run_status(
|
||||
transcript.workflow_run_id
|
||||
)
|
||||
print(f" Status: {status.value}", file=sys.stderr)
|
||||
|
||||
if status == V1TaskStatus.COMPLETED:
|
||||
print("Workflow completed successfully", file=sys.stderr)
|
||||
break
|
||||
elif status in (V1TaskStatus.FAILED, V1TaskStatus.CANCELLED):
|
||||
print(f"Workflow failed: {status}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
await asyncio.sleep(5)
|
||||
else:
|
||||
print(
|
||||
"Task dispatched (use --sync to wait for completion)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
elif sync:
|
||||
print("Waiting for task completion...", file=sys.stderr)
|
||||
while not result.ready():
|
||||
print(f" Status: {result.state}", file=sys.stderr)
|
||||
time.sleep(5)
|
||||
|
||||
if result.successful():
|
||||
print("Task completed successfully", file=sys.stderr)
|
||||
else:
|
||||
print(f"Task failed: {result.result}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
await asyncio.sleep(5)
|
||||
else:
|
||||
print(
|
||||
"Task dispatched (use --sync to wait for completion)", file=sys.stderr
|
||||
"Task dispatched (use --sync to wait for completion)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
finally:
|
||||
|
||||
@@ -89,14 +89,16 @@ class StartRecordingRequest(BaseModel):
|
||||
|
||||
@router.post("/meetings/{meeting_id}/recordings/start")
|
||||
async def start_recording(
|
||||
meeting_id: NonEmptyString, body: StartRecordingRequest
|
||||
meeting_id: NonEmptyString,
|
||||
body: StartRecordingRequest,
|
||||
user: Annotated[
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional_if_public_mode)
|
||||
],
|
||||
) -> dict[str, Any]:
|
||||
"""Start cloud or raw-tracks recording via Daily.co REST API.
|
||||
|
||||
Both cloud and raw-tracks are started via REST API to bypass enable_recording limitation of allowing only 1 recording at a time.
|
||||
Uses different instanceIds for cloud vs raw-tracks (same won't work)
|
||||
|
||||
Note: No authentication required - anonymous users supported. TODO this is a DOS vector
|
||||
"""
|
||||
meeting = await meetings_controller.get_by_id(meeting_id)
|
||||
if not meeting:
|
||||
|
||||
@@ -17,7 +17,6 @@ from reflector.db.rooms import rooms_controller
|
||||
from reflector.redis_cache import RedisAsyncLock
|
||||
from reflector.schemas.platform import Platform
|
||||
from reflector.services.ics_sync import ics_sync_service
|
||||
from reflector.settings import settings
|
||||
from reflector.utils.url import add_query_param
|
||||
from reflector.video_platforms.factory import create_platform_client
|
||||
from reflector.worker.webhook import test_webhook
|
||||
@@ -178,11 +177,10 @@ router = APIRouter()
|
||||
|
||||
@router.get("/rooms", response_model=Page[RoomDetails])
|
||||
async def rooms_list(
|
||||
user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)],
|
||||
user: Annotated[
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional_if_public_mode)
|
||||
],
|
||||
) -> list[RoomDetails]:
|
||||
if not user and not settings.PUBLIC_MODE:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
|
||||
user_id = user["sub"] if user else None
|
||||
|
||||
paginated = await apaginate(
|
||||
|
||||
@@ -263,16 +263,15 @@ class SearchResponse(BaseModel):
|
||||
|
||||
@router.get("/transcripts", response_model=Page[GetTranscriptMinimal])
|
||||
async def transcripts_list(
|
||||
user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)],
|
||||
user: Annotated[
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional_if_public_mode)
|
||||
],
|
||||
source_kind: SourceKind | None = None,
|
||||
room_id: str | None = None,
|
||||
search_term: str | None = None,
|
||||
change_seq_from: int | None = None,
|
||||
sort_by: Literal["created_at", "change_seq"] | None = None,
|
||||
):
|
||||
if not user and not settings.PUBLIC_MODE:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
|
||||
user_id = user["sub"] if user else None
|
||||
|
||||
# Default behavior preserved: sort_by=None → "-created_at"
|
||||
@@ -307,13 +306,10 @@ async def transcripts_search(
|
||||
from_datetime: SearchFromDatetimeParam = None,
|
||||
to_datetime: SearchToDatetimeParam = None,
|
||||
user: Annotated[
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional)
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional_if_public_mode)
|
||||
] = None,
|
||||
):
|
||||
"""Full-text search across transcript titles and content."""
|
||||
if not user and not settings.PUBLIC_MODE:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
|
||||
user_id = user["sub"] if user else None
|
||||
|
||||
if from_datetime and to_datetime and from_datetime > to_datetime:
|
||||
@@ -346,7 +342,9 @@ async def transcripts_search(
|
||||
@router.post("/transcripts", response_model=GetTranscriptWithParticipants)
|
||||
async def transcripts_create(
|
||||
info: CreateTranscript,
|
||||
user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)],
|
||||
user: Annotated[
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional_if_public_mode)
|
||||
],
|
||||
):
|
||||
user_id = user["sub"] if user else None
|
||||
transcript = await transcripts_controller.add(
|
||||
|
||||
@@ -62,8 +62,7 @@ async def transcript_add_participant(
|
||||
transcript = await transcripts_controller.get_by_id_for_http(
|
||||
transcript_id, user_id=user_id
|
||||
)
|
||||
if transcript.user_id is not None and transcript.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
transcripts_controller.check_can_mutate(transcript, user_id)
|
||||
|
||||
# ensure the speaker is unique
|
||||
if participant.speaker is not None and transcript.participants is not None:
|
||||
@@ -109,8 +108,7 @@ async def transcript_update_participant(
|
||||
transcript = await transcripts_controller.get_by_id_for_http(
|
||||
transcript_id, user_id=user_id
|
||||
)
|
||||
if transcript.user_id is not None and transcript.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
transcripts_controller.check_can_mutate(transcript, user_id)
|
||||
|
||||
# ensure the speaker is unique
|
||||
for p in transcript.participants:
|
||||
@@ -148,7 +146,6 @@ async def transcript_delete_participant(
|
||||
transcript = await transcripts_controller.get_by_id_for_http(
|
||||
transcript_id, user_id=user_id
|
||||
)
|
||||
if transcript.user_id is not None and transcript.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
transcripts_controller.check_can_mutate(transcript, user_id)
|
||||
await transcripts_controller.delete_participant(transcript, participant_id)
|
||||
return DeletionStatus(status="ok")
|
||||
|
||||
@@ -26,7 +26,9 @@ class ProcessStatus(BaseModel):
|
||||
@router.post("/transcripts/{transcript_id}/process")
|
||||
async def transcript_process(
|
||||
transcript_id: str,
|
||||
user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)],
|
||||
user: Annotated[
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional_if_public_mode)
|
||||
],
|
||||
) -> ProcessStatus:
|
||||
user_id = user["sub"] if user else None
|
||||
transcript = await transcripts_controller.get_by_id_for_http(
|
||||
@@ -50,8 +52,5 @@ async def transcript_process(
|
||||
if isinstance(config, ProcessError):
|
||||
raise HTTPException(status_code=500, detail=config.detail)
|
||||
else:
|
||||
# When transcript is in error state, force a new workflow instead of replaying
|
||||
# (replay would re-run from failure point with same conditions and likely fail again)
|
||||
force = transcript.status == "error"
|
||||
await dispatch_transcript_processing(config, force=force)
|
||||
await dispatch_transcript_processing(config, force=True)
|
||||
return ProcessStatus(status="ok")
|
||||
|
||||
@@ -41,8 +41,7 @@ async def transcript_assign_speaker(
|
||||
transcript = await transcripts_controller.get_by_id_for_http(
|
||||
transcript_id, user_id=user_id
|
||||
)
|
||||
if transcript.user_id is not None and transcript.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
transcripts_controller.check_can_mutate(transcript, user_id)
|
||||
|
||||
if not transcript:
|
||||
raise HTTPException(status_code=404, detail="Transcript not found")
|
||||
@@ -121,8 +120,7 @@ async def transcript_merge_speaker(
|
||||
transcript = await transcripts_controller.get_by_id_for_http(
|
||||
transcript_id, user_id=user_id
|
||||
)
|
||||
if transcript.user_id is not None and transcript.user_id != user_id:
|
||||
raise HTTPException(status_code=403, detail="Not authorized")
|
||||
transcripts_controller.check_can_mutate(transcript, user_id)
|
||||
|
||||
if not transcript:
|
||||
raise HTTPException(status_code=404, detail="Transcript not found")
|
||||
|
||||
@@ -6,7 +6,7 @@ from pydantic import BaseModel
|
||||
|
||||
import reflector.auth as auth
|
||||
from reflector.db.transcripts import SourceKind, transcripts_controller
|
||||
from reflector.pipelines.main_file_pipeline import task_pipeline_file_process
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -21,7 +21,9 @@ async def transcript_record_upload(
|
||||
chunk_number: int,
|
||||
total_chunks: int,
|
||||
chunk: UploadFile,
|
||||
user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)],
|
||||
user: Annotated[
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional_if_public_mode)
|
||||
],
|
||||
):
|
||||
user_id = user["sub"] if user else None
|
||||
transcript = await transcripts_controller.get_by_id_for_http(
|
||||
@@ -93,7 +95,14 @@ async def transcript_record_upload(
|
||||
transcript, {"status": "uploaded", "source_kind": SourceKind.FILE}
|
||||
)
|
||||
|
||||
# launch a background task to process the file
|
||||
task_pipeline_file_process.delay(transcript_id=transcript_id)
|
||||
# launch Hatchet workflow to process the file
|
||||
workflow_id = await HatchetClientManager.start_workflow(
|
||||
"FilePipeline",
|
||||
{"transcript_id": str(transcript_id)},
|
||||
additional_metadata={"transcript_id": str(transcript_id)},
|
||||
)
|
||||
|
||||
# Save workflow_run_id for duplicate detection and status polling
|
||||
await transcripts_controller.update(transcript, {"workflow_run_id": workflow_id})
|
||||
|
||||
return UploadStatus(status="ok")
|
||||
|
||||
@@ -15,7 +15,9 @@ async def transcript_record_webrtc(
|
||||
transcript_id: str,
|
||||
params: RtcOffer,
|
||||
request: Request,
|
||||
user: Annotated[Optional[auth.UserInfo], Depends(auth.current_user_optional)],
|
||||
user: Annotated[
|
||||
Optional[auth.UserInfo], Depends(auth.current_user_optional_if_public_mode)
|
||||
],
|
||||
):
|
||||
user_id = user["sub"] if user else None
|
||||
transcript = await transcripts_controller.get_by_id_for_http(
|
||||
|
||||
@@ -25,7 +25,6 @@ from reflector.db.transcripts import (
|
||||
transcripts_controller,
|
||||
)
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
from reflector.pipelines.main_file_pipeline import task_pipeline_file_process
|
||||
from reflector.pipelines.main_live_pipeline import asynctask
|
||||
from reflector.pipelines.topic_processing import EmptyPipeline
|
||||
from reflector.processors import AudioFileWriterProcessor
|
||||
@@ -132,7 +131,7 @@ async def process_recording(bucket_name: str, object_key: str):
|
||||
target_language="en",
|
||||
user_id=room.user_id,
|
||||
recording_id=recording.id,
|
||||
share_mode="public",
|
||||
share_mode="semi-private",
|
||||
meeting_id=meeting.id,
|
||||
room_id=room.id,
|
||||
)
|
||||
@@ -163,7 +162,14 @@ async def process_recording(bucket_name: str, object_key: str):
|
||||
|
||||
await transcripts_controller.update(transcript, {"status": "uploaded"})
|
||||
|
||||
task_pipeline_file_process.delay(transcript_id=transcript.id)
|
||||
await HatchetClientManager.start_workflow(
|
||||
"FilePipeline",
|
||||
{
|
||||
"transcript_id": str(transcript.id),
|
||||
"room_id": str(room.id) if room else None,
|
||||
},
|
||||
additional_metadata={"transcript_id": str(transcript.id)},
|
||||
)
|
||||
|
||||
|
||||
@shared_task
|
||||
@@ -343,7 +349,7 @@ async def _process_multitrack_recording_inner(
|
||||
target_language="en",
|
||||
user_id=room.user_id,
|
||||
recording_id=recording.id,
|
||||
share_mode="public",
|
||||
share_mode="semi-private",
|
||||
meeting_id=meeting.id,
|
||||
room_id=room.id,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -437,6 +437,8 @@ async def ws_manager_in_memory(monkeypatch):
|
||||
|
||||
try:
|
||||
fastapi_app.dependency_overrides[auth.current_user_optional] = lambda: None
|
||||
# current_user_optional_if_public_mode is NOT overridden here so the real
|
||||
# implementation runs and enforces the PUBLIC_MODE check during tests.
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -491,37 +493,39 @@ async def authenticated_client2():
|
||||
@asynccontextmanager
|
||||
async def authenticated_client_ctx():
|
||||
from reflector.app import app
|
||||
from reflector.auth import current_user, current_user_optional
|
||||
from reflector.auth import (
|
||||
current_user,
|
||||
current_user_optional,
|
||||
current_user_optional_if_public_mode,
|
||||
)
|
||||
|
||||
app.dependency_overrides[current_user] = lambda: {
|
||||
"sub": "randomuserid",
|
||||
"email": "test@mail.com",
|
||||
}
|
||||
app.dependency_overrides[current_user_optional] = lambda: {
|
||||
"sub": "randomuserid",
|
||||
"email": "test@mail.com",
|
||||
}
|
||||
_user = lambda: {"sub": "randomuserid", "email": "test@mail.com"}
|
||||
app.dependency_overrides[current_user] = _user
|
||||
app.dependency_overrides[current_user_optional] = _user
|
||||
app.dependency_overrides[current_user_optional_if_public_mode] = _user
|
||||
yield
|
||||
del app.dependency_overrides[current_user]
|
||||
del app.dependency_overrides[current_user_optional]
|
||||
del app.dependency_overrides[current_user_optional_if_public_mode]
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def authenticated_client2_ctx():
|
||||
from reflector.app import app
|
||||
from reflector.auth import current_user, current_user_optional
|
||||
from reflector.auth import (
|
||||
current_user,
|
||||
current_user_optional,
|
||||
current_user_optional_if_public_mode,
|
||||
)
|
||||
|
||||
app.dependency_overrides[current_user] = lambda: {
|
||||
"sub": "randomuserid2",
|
||||
"email": "test@mail.com",
|
||||
}
|
||||
app.dependency_overrides[current_user_optional] = lambda: {
|
||||
"sub": "randomuserid2",
|
||||
"email": "test@mail.com",
|
||||
}
|
||||
_user = lambda: {"sub": "randomuserid2", "email": "test@mail.com"}
|
||||
app.dependency_overrides[current_user] = _user
|
||||
app.dependency_overrides[current_user_optional] = _user
|
||||
app.dependency_overrides[current_user_optional_if_public_mode] = _user
|
||||
yield
|
||||
del app.dependency_overrides[current_user]
|
||||
del app.dependency_overrides[current_user_optional]
|
||||
del app.dependency_overrides[current_user_optional_if_public_mode]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
@@ -534,23 +538,64 @@ def fake_mp3_upload():
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_hatchet_client():
|
||||
"""Reset HatchetClientManager singleton before and after each test.
|
||||
def mock_hatchet_client():
|
||||
"""Mock HatchetClientManager for all tests.
|
||||
|
||||
This ensures test isolation - each test starts with a fresh client state.
|
||||
The fixture is autouse=True so it applies to all tests automatically.
|
||||
Prevents tests from connecting to a real Hatchet server. The dummy token
|
||||
in [tool.pytest_env] prevents the import-time ValueError, but the SDK
|
||||
would still try to connect when get_client() is called. This fixture
|
||||
mocks get_client to return a MagicMock and start_workflow to return a
|
||||
dummy workflow ID.
|
||||
"""
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
|
||||
# Reset before test
|
||||
HatchetClientManager.reset()
|
||||
yield
|
||||
# Reset after test to clean up
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_client.workflow.return_value = MagicMock()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
HatchetClientManager,
|
||||
"get_client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch.object(
|
||||
HatchetClientManager,
|
||||
"start_workflow",
|
||||
new_callable=AsyncMock,
|
||||
return_value="mock-workflow-id",
|
||||
),
|
||||
patch.object(
|
||||
HatchetClientManager,
|
||||
"get_workflow_run_status",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
patch.object(
|
||||
HatchetClientManager,
|
||||
"can_replay",
|
||||
new_callable=AsyncMock,
|
||||
return_value=False,
|
||||
),
|
||||
patch.object(
|
||||
HatchetClientManager,
|
||||
"cancel_workflow",
|
||||
new_callable=AsyncMock,
|
||||
),
|
||||
patch.object(
|
||||
HatchetClientManager,
|
||||
"replay_workflow",
|
||||
new_callable=AsyncMock,
|
||||
),
|
||||
):
|
||||
yield mock_client
|
||||
|
||||
HatchetClientManager.reset()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fake_transcript_with_topics(tmpdir, client):
|
||||
async def fake_transcript_with_topics(tmpdir, client, monkeypatch):
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
@@ -559,6 +604,9 @@ async def fake_transcript_with_topics(tmpdir, client):
|
||||
from reflector.settings import settings
|
||||
from reflector.views.transcripts import transcripts_controller
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
settings.DATA_DIR = Path(tmpdir)
|
||||
|
||||
# create a transcript
|
||||
|
||||
218
server/tests/docker-compose.integration.yml
Normal file
218
server/tests/docker-compose.integration.yml
Normal file
@@ -0,0 +1,218 @@
|
||||
# Integration test stack — full pipeline end-to-end.
|
||||
#
|
||||
# Usage:
|
||||
# docker compose -f server/tests/docker-compose.integration.yml up -d --build
|
||||
#
|
||||
# Requires .env.integration in the repo root (generated by CI workflow).
|
||||
|
||||
x-backend-env: &backend-env
|
||||
DATABASE_URL: postgresql+asyncpg://reflector:reflector@postgres:5432/reflector
|
||||
REDIS_HOST: redis
|
||||
CELERY_BROKER_URL: redis://redis:6379/1
|
||||
CELERY_RESULT_BACKEND: redis://redis:6379/1
|
||||
HATCHET_CLIENT_TOKEN: ${HATCHET_CLIENT_TOKEN:-}
|
||||
HATCHET_CLIENT_SERVER_URL: http://hatchet:8888
|
||||
HATCHET_CLIENT_HOST_PORT: hatchet:7077
|
||||
HATCHET_CLIENT_TLS_STRATEGY: none
|
||||
# ML backends — CPU-only, no external services
|
||||
TRANSCRIPT_BACKEND: whisper
|
||||
WHISPER_CHUNK_MODEL: tiny
|
||||
WHISPER_FILE_MODEL: tiny
|
||||
DIARIZATION_BACKEND: pyannote
|
||||
TRANSLATION_BACKEND: passthrough
|
||||
# Storage — local Garage S3
|
||||
TRANSCRIPT_STORAGE_BACKEND: aws
|
||||
TRANSCRIPT_STORAGE_AWS_ENDPOINT_URL: http://garage:3900
|
||||
TRANSCRIPT_STORAGE_AWS_BUCKET_NAME: reflector-media
|
||||
TRANSCRIPT_STORAGE_AWS_REGION: garage
|
||||
# Daily mock
|
||||
DAILY_API_URL: http://mock-daily:8080/v1
|
||||
DAILY_API_KEY: fake-daily-key
|
||||
# Auth
|
||||
PUBLIC_MODE: "true"
|
||||
AUTH_BACKEND: none
|
||||
# LLM (injected from CI)
|
||||
LLM_URL: ${LLM_URL:-}
|
||||
LLM_API_KEY: ${LLM_API_KEY:-}
|
||||
LLM_MODEL: ${LLM_MODEL:-gpt-4o-mini}
|
||||
# HuggingFace (for pyannote gated models)
|
||||
HF_TOKEN: ${HF_TOKEN:-}
|
||||
# Garage S3 credentials — hardcoded test keys, containers are ephemeral
|
||||
TRANSCRIPT_STORAGE_AWS_ACCESS_KEY_ID: GK0123456789abcdef01234567 # gitleaks:allow
|
||||
TRANSCRIPT_STORAGE_AWS_SECRET_ACCESS_KEY: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" # gitleaks:allow
|
||||
# NOTE: DAILYCO_STORAGE_AWS_* intentionally NOT set — forces fallback to
|
||||
# get_transcripts_storage() which has ENDPOINT_URL pointing at Garage.
|
||||
# Setting them would bypass the endpoint and generate presigned URLs for AWS.
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
command: ["postgres", "-c", "max_connections=200"]
|
||||
environment:
|
||||
POSTGRES_USER: reflector
|
||||
POSTGRES_PASSWORD: reflector
|
||||
POSTGRES_DB: reflector
|
||||
volumes:
|
||||
- ../../server/docker/init-hatchet-db.sql:/docker-entrypoint-initdb.d/init-hatchet-db.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U reflector"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
redis:
|
||||
image: redis:7.2-alpine
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
hatchet:
|
||||
image: ghcr.io/hatchet-dev/hatchet/hatchet-lite:latest
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: "postgresql://reflector:reflector@postgres:5432/hatchet?sslmode=disable&connect_timeout=30"
|
||||
SERVER_AUTH_COOKIE_INSECURE: "t"
|
||||
SERVER_AUTH_COOKIE_DOMAIN: "localhost"
|
||||
SERVER_GRPC_BIND_ADDRESS: "0.0.0.0"
|
||||
SERVER_GRPC_INSECURE: "t"
|
||||
SERVER_GRPC_BROADCAST_ADDRESS: hatchet:7077
|
||||
SERVER_GRPC_PORT: "7077"
|
||||
SERVER_AUTH_SET_EMAIL_VERIFIED: "t"
|
||||
SERVER_INTERNAL_CLIENT_INTERNAL_GRPC_BROADCAST_ADDRESS: hatchet:7077
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8888/api/live"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 15
|
||||
start_period: 30s
|
||||
|
||||
garage:
|
||||
image: dxflrs/garage:v1.1.0
|
||||
volumes:
|
||||
- ./integration/garage.toml:/etc/garage.toml:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "/garage", "stats"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
|
||||
mock-daily:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: integration/Dockerfile.mock-daily
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/v1/recordings/test')"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
server:
|
||||
build:
|
||||
context: ../../server
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
<<: *backend-env
|
||||
ENTRYPOINT: server
|
||||
WEBRTC_HOST: server
|
||||
WEBRTC_PORT_RANGE: "52000-52100"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
hatchet:
|
||||
condition: service_healthy
|
||||
garage:
|
||||
condition: service_healthy
|
||||
mock-daily:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- server_data:/app/data
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: ../../server
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
<<: *backend-env
|
||||
ENTRYPOINT: worker
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- server_data:/app/data
|
||||
|
||||
hatchet-worker-cpu:
|
||||
build:
|
||||
context: ../../server
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
<<: *backend-env
|
||||
ENTRYPOINT: hatchet-worker-cpu
|
||||
depends_on:
|
||||
hatchet:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- server_data:/app/data
|
||||
|
||||
hatchet-worker-llm:
|
||||
build:
|
||||
context: ../../server
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
<<: *backend-env
|
||||
ENTRYPOINT: hatchet-worker-llm
|
||||
depends_on:
|
||||
hatchet:
|
||||
condition: service_healthy
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- server_data:/app/data
|
||||
|
||||
test-runner:
|
||||
build:
|
||||
context: ../../server
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
<<: *backend-env
|
||||
# Override DATABASE_URL for sync driver (used by direct DB access in tests)
|
||||
DATABASE_URL_ASYNC: postgresql+asyncpg://reflector:reflector@postgres:5432/reflector
|
||||
DATABASE_URL: postgresql+asyncpg://reflector:reflector@postgres:5432/reflector
|
||||
SERVER_URL: http://server:1250
|
||||
GARAGE_ENDPOINT: http://garage:3900
|
||||
depends_on:
|
||||
server:
|
||||
condition: service_started
|
||||
worker:
|
||||
condition: service_started
|
||||
hatchet-worker-cpu:
|
||||
condition: service_started
|
||||
hatchet-worker-llm:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- server_data:/app/data
|
||||
# Mount test files into the container
|
||||
- ./records:/app/tests/records:ro
|
||||
- ./integration:/app/tests/integration:ro
|
||||
entrypoint: ["sleep", "infinity"]
|
||||
|
||||
volumes:
|
||||
server_data:
|
||||
|
||||
networks:
|
||||
default:
|
||||
attachable: true
|
||||
9
server/tests/integration/Dockerfile.mock-daily
Normal file
9
server/tests/integration/Dockerfile.mock-daily
Normal file
@@ -0,0 +1,9 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
RUN pip install --no-cache-dir fastapi uvicorn[standard]
|
||||
|
||||
WORKDIR /app
|
||||
COPY integration/mock_daily_server.py /app/mock_daily_server.py
|
||||
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "mock_daily_server:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
0
server/tests/integration/__init__.py
Normal file
0
server/tests/integration/__init__.py
Normal file
116
server/tests/integration/conftest.py
Normal file
116
server/tests/integration/conftest.py
Normal file
@@ -0,0 +1,116 @@
|
||||
"""
|
||||
Integration test fixtures — no mocks, real services.
|
||||
|
||||
All services (PostgreSQL, Redis, Hatchet, Garage, server, workers) are
|
||||
expected to be running via docker-compose.integration.yml.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
SERVER_URL = os.environ.get("SERVER_URL", "http://server:1250")
|
||||
GARAGE_ENDPOINT = os.environ.get("GARAGE_ENDPOINT", "http://garage:3900")
|
||||
DATABASE_URL = os.environ.get(
|
||||
"DATABASE_URL_ASYNC",
|
||||
os.environ.get(
|
||||
"DATABASE_URL",
|
||||
"postgresql+asyncpg://reflector:reflector@postgres:5432/reflector",
|
||||
),
|
||||
)
|
||||
GARAGE_KEY_ID = os.environ.get("TRANSCRIPT_STORAGE_AWS_ACCESS_KEY_ID", "")
|
||||
GARAGE_KEY_SECRET = os.environ.get("TRANSCRIPT_STORAGE_AWS_SECRET_ACCESS_KEY", "")
|
||||
BUCKET_NAME = "reflector-media"
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def api_client():
|
||||
"""HTTP client pointed at the running server."""
|
||||
async with httpx.AsyncClient(
|
||||
base_url=f"{SERVER_URL}/v1",
|
||||
timeout=httpx.Timeout(30.0),
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def s3_client():
|
||||
"""Boto3 S3 client pointed at Garage."""
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=GARAGE_ENDPOINT,
|
||||
aws_access_key_id=GARAGE_KEY_ID,
|
||||
aws_secret_access_key=GARAGE_KEY_SECRET,
|
||||
region_name="garage",
|
||||
)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def db_engine():
|
||||
"""SQLAlchemy async engine for direct DB operations."""
|
||||
engine = create_async_engine(DATABASE_URL)
|
||||
yield engine
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def test_records_dir():
|
||||
"""Path to the test audio files directory."""
|
||||
return Path(__file__).parent.parent / "records"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def bucket_name():
|
||||
"""S3 bucket name used for integration tests."""
|
||||
return BUCKET_NAME
|
||||
|
||||
|
||||
async def _poll_transcript_status(
|
||||
client: httpx.AsyncClient,
|
||||
transcript_id: str,
|
||||
target: str | tuple[str, ...],
|
||||
error: str = "error",
|
||||
max_wait: int = 300,
|
||||
interval: int = 3,
|
||||
) -> dict:
|
||||
"""
|
||||
Poll GET /transcripts/{id} until status matches target or error.
|
||||
|
||||
target can be a single status string or a tuple of acceptable statuses.
|
||||
Returns the transcript dict on success, raises on timeout or error status.
|
||||
"""
|
||||
targets = (target,) if isinstance(target, str) else target
|
||||
elapsed = 0
|
||||
status = None
|
||||
while elapsed < max_wait:
|
||||
resp = await client.get(f"/transcripts/{transcript_id}")
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
status = data.get("status")
|
||||
|
||||
if status in targets:
|
||||
return data
|
||||
if status == error:
|
||||
raise AssertionError(
|
||||
f"Transcript {transcript_id} reached error status: {data}"
|
||||
)
|
||||
|
||||
await asyncio.sleep(interval)
|
||||
elapsed += interval
|
||||
|
||||
raise TimeoutError(
|
||||
f"Transcript {transcript_id} did not reach status '{target}' "
|
||||
f"within {max_wait}s (last status: {status})"
|
||||
)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def poll_transcript_status():
|
||||
"""Returns the poll_transcript_status async helper function."""
|
||||
return _poll_transcript_status
|
||||
14
server/tests/integration/garage.toml
Normal file
14
server/tests/integration/garage.toml
Normal file
@@ -0,0 +1,14 @@
|
||||
metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
replication_factor = 1
|
||||
|
||||
rpc_secret = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" # gitleaks:allow
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
|
||||
[s3_api]
|
||||
api_bind_addr = "[::]:3900"
|
||||
s3_region = "garage"
|
||||
root_domain = ".s3.garage.localhost"
|
||||
|
||||
[admin]
|
||||
api_bind_addr = "[::]:3903"
|
||||
62
server/tests/integration/garage_setup.sh
Executable file
62
server/tests/integration/garage_setup.sh
Executable file
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Initialize Garage bucket and keys for integration tests.
|
||||
# Run inside the Garage container after it's healthy.
|
||||
#
|
||||
# Outputs KEY_ID and KEY_SECRET to stdout (last two lines).
|
||||
#
|
||||
# Note: uses /bin/sh (not bash) since the Garage container is minimal.
|
||||
#
|
||||
set -eu
|
||||
|
||||
echo "Waiting for Garage to be ready..."
|
||||
i=0
|
||||
while [ "$i" -lt 30 ]; do
|
||||
if /garage stats >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
# Layout setup
|
||||
NODE_ID=$(/garage node id -q | tr -d '[:space:]')
|
||||
LAYOUT_STATUS=$(/garage layout show 2>&1 || true)
|
||||
if echo "$LAYOUT_STATUS" | grep -q "No nodes"; then
|
||||
/garage layout assign "$NODE_ID" -c 1G -z dc1
|
||||
/garage layout apply --version 1
|
||||
echo "Layout applied."
|
||||
else
|
||||
echo "Layout already configured."
|
||||
fi
|
||||
|
||||
# Bucket
|
||||
if ! /garage bucket info reflector-media >/dev/null 2>&1; then
|
||||
/garage bucket create reflector-media
|
||||
echo "Bucket 'reflector-media' created."
|
||||
else
|
||||
echo "Bucket 'reflector-media' already exists."
|
||||
fi
|
||||
|
||||
# Key
|
||||
if /garage key info reflector-test >/dev/null 2>&1; then
|
||||
echo "Key 'reflector-test' already exists."
|
||||
KEY_OUTPUT=$(/garage key info reflector-test 2>&1)
|
||||
else
|
||||
KEY_OUTPUT=$(/garage key create reflector-test 2>&1)
|
||||
echo "Key 'reflector-test' created."
|
||||
fi
|
||||
|
||||
# Permissions
|
||||
/garage bucket allow reflector-media --read --write --key reflector-test
|
||||
|
||||
# Extract key ID and secret from output using POSIX-compatible parsing
|
||||
# garage key output format:
|
||||
# Key name: reflector-test
|
||||
# Key ID: GK...
|
||||
# Secret key: ...
|
||||
KEY_ID=$(echo "$KEY_OUTPUT" | grep "Key ID" | sed 's/.*Key ID: *//')
|
||||
KEY_SECRET=$(echo "$KEY_OUTPUT" | grep "Secret key" | sed 's/.*Secret key: *//')
|
||||
|
||||
echo "GARAGE_KEY_ID=${KEY_ID}"
|
||||
echo "GARAGE_KEY_SECRET=${KEY_SECRET}"
|
||||
75
server/tests/integration/mock_daily_server.py
Normal file
75
server/tests/integration/mock_daily_server.py
Normal file
@@ -0,0 +1,75 @@
|
||||
"""
|
||||
Minimal FastAPI mock for Daily.co API.
|
||||
|
||||
Serves canned responses for:
|
||||
- GET /v1/recordings/{recording_id}
|
||||
- GET /v1/meetings/{meeting_id}/participants
|
||||
"""
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
app = FastAPI(title="Mock Daily API")
|
||||
|
||||
|
||||
# Participant UUIDs must be 36-char hex UUIDs to match Daily's filename format
|
||||
PARTICIPANT_A_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
|
||||
PARTICIPANT_B_ID = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
|
||||
|
||||
# Daily-format track keys: {recording_start_ts}-{participant_id}-cam-audio-{track_start_ts}
|
||||
TRACK_KEYS = [
|
||||
f"1700000000000-{PARTICIPANT_A_ID}-cam-audio-1700000001000",
|
||||
f"1700000000000-{PARTICIPANT_B_ID}-cam-audio-1700000001000",
|
||||
]
|
||||
|
||||
|
||||
@app.get("/v1/recordings/{recording_id}")
|
||||
async def get_recording(recording_id: str):
|
||||
return {
|
||||
"id": recording_id,
|
||||
"room_name": "integration-test-room",
|
||||
"start_ts": 1700000000,
|
||||
"type": "raw-tracks",
|
||||
"status": "finished",
|
||||
"max_participants": 2,
|
||||
"duration": 5,
|
||||
"share_token": None,
|
||||
"s3": {
|
||||
"bucket_name": "reflector-media",
|
||||
"bucket_region": "garage",
|
||||
"key": None,
|
||||
"endpoint": None,
|
||||
},
|
||||
"s3key": None,
|
||||
"tracks": [
|
||||
{"type": "audio", "s3Key": key, "size": 100000} for key in TRACK_KEYS
|
||||
],
|
||||
"mtgSessionId": "mock-mtg-session-id",
|
||||
}
|
||||
|
||||
|
||||
@app.get("/v1/meetings/{meeting_id}/participants")
|
||||
async def get_meeting_participants(meeting_id: str):
|
||||
return {
|
||||
"data": [
|
||||
{
|
||||
"user_id": "user-a",
|
||||
"participant_id": PARTICIPANT_A_ID,
|
||||
"user_name": "Speaker A",
|
||||
"join_time": 1700000000,
|
||||
"duration": 300,
|
||||
},
|
||||
{
|
||||
"user_id": "user-b",
|
||||
"participant_id": PARTICIPANT_B_ID,
|
||||
"user_name": "Speaker B",
|
||||
"join_time": 1700000010,
|
||||
"duration": 290,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=8080)
|
||||
61
server/tests/integration/test_file_pipeline.py
Normal file
61
server/tests/integration/test_file_pipeline.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
Integration test: File upload → FilePipeline → full processing.
|
||||
|
||||
Exercises: upload endpoint → Hatchet FilePipeline → whisper transcription →
|
||||
pyannote diarization → LLM summarization/topics → status "ended".
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_pipeline_end_to_end(
|
||||
api_client, test_records_dir, poll_transcript_status
|
||||
):
|
||||
"""Upload a WAV file and verify the full pipeline completes."""
|
||||
# 1. Create transcript
|
||||
resp = await api_client.post(
|
||||
"/transcripts",
|
||||
json={"name": "integration-file-test", "source_kind": "file"},
|
||||
)
|
||||
assert resp.status_code == 200, f"Failed to create transcript: {resp.text}"
|
||||
transcript = resp.json()
|
||||
transcript_id = transcript["id"]
|
||||
|
||||
# 2. Upload audio file (single chunk)
|
||||
audio_path = test_records_dir / "test_short.wav"
|
||||
assert audio_path.exists(), f"Test audio file not found: {audio_path}"
|
||||
|
||||
with open(audio_path, "rb") as f:
|
||||
resp = await api_client.post(
|
||||
f"/transcripts/{transcript_id}/record/upload",
|
||||
params={"chunk_number": 0, "total_chunks": 1},
|
||||
files={"chunk": ("test_short.wav", f, "audio/wav")},
|
||||
)
|
||||
assert resp.status_code == 200, f"Upload failed: {resp.text}"
|
||||
|
||||
# 3. Poll until pipeline completes
|
||||
data = await poll_transcript_status(
|
||||
api_client, transcript_id, target="ended", max_wait=300
|
||||
)
|
||||
|
||||
# 4. Assertions
|
||||
assert data["status"] == "ended"
|
||||
assert data.get("title") and len(data["title"]) > 0, "Title should be non-empty"
|
||||
assert (
|
||||
data.get("long_summary") and len(data["long_summary"]) > 0
|
||||
), "Long summary should be non-empty"
|
||||
assert (
|
||||
data.get("short_summary") and len(data["short_summary"]) > 0
|
||||
), "Short summary should be non-empty"
|
||||
|
||||
# Topics are served from a separate endpoint
|
||||
topics_resp = await api_client.get(f"/transcripts/{transcript_id}/topics")
|
||||
assert topics_resp.status_code == 200, f"Failed to get topics: {topics_resp.text}"
|
||||
topics = topics_resp.json()
|
||||
assert len(topics) >= 1, "Should have at least 1 topic"
|
||||
for topic in topics:
|
||||
assert topic.get("title"), "Each topic should have a title"
|
||||
assert topic.get("summary"), "Each topic should have a summary"
|
||||
|
||||
assert data.get("duration", 0) > 0, "Duration should be positive"
|
||||
109
server/tests/integration/test_live_pipeline.py
Normal file
109
server/tests/integration/test_live_pipeline.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Integration test: WebRTC stream → LivePostProcessingPipeline → full processing.
|
||||
|
||||
Exercises: WebRTC SDP exchange → live audio streaming → connection close →
|
||||
Hatchet LivePostPipeline → whisper transcription → LLM summarization/topics → status "ended".
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from aiortc import RTCPeerConnection, RTCSessionDescription
|
||||
from aiortc.contrib.media import MediaPlayer
|
||||
|
||||
SERVER_URL = os.environ.get("SERVER_URL", "http://server:1250")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_pipeline_end_to_end(
|
||||
api_client, test_records_dir, poll_transcript_status
|
||||
):
|
||||
"""Stream audio via WebRTC and verify the full post-processing pipeline completes."""
|
||||
# 1. Create transcript
|
||||
resp = await api_client.post(
|
||||
"/transcripts",
|
||||
json={"name": "integration-live-test"},
|
||||
)
|
||||
assert resp.status_code == 200, f"Failed to create transcript: {resp.text}"
|
||||
transcript = resp.json()
|
||||
transcript_id = transcript["id"]
|
||||
|
||||
# 2. Set up WebRTC peer connection with audio from test file
|
||||
audio_path = test_records_dir / "test_short.wav"
|
||||
assert audio_path.exists(), f"Test audio file not found: {audio_path}"
|
||||
|
||||
pc = RTCPeerConnection()
|
||||
player = MediaPlayer(audio_path.as_posix())
|
||||
|
||||
# Add audio track
|
||||
audio_track = player.audio
|
||||
pc.addTrack(audio_track)
|
||||
|
||||
# Create data channel (server expects this for STOP command)
|
||||
channel = pc.createDataChannel("data-channel")
|
||||
|
||||
# 3. Generate SDP offer
|
||||
offer = await pc.createOffer()
|
||||
await pc.setLocalDescription(offer)
|
||||
|
||||
sdp_payload = {
|
||||
"sdp": pc.localDescription.sdp,
|
||||
"type": pc.localDescription.type,
|
||||
}
|
||||
|
||||
# 4. Send offer to server and get answer
|
||||
webrtc_url = f"{SERVER_URL}/v1/transcripts/{transcript_id}/record/webrtc"
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(30.0)) as client:
|
||||
resp = await client.post(webrtc_url, json=sdp_payload)
|
||||
assert resp.status_code == 200, f"WebRTC offer failed: {resp.text}"
|
||||
|
||||
answer_data = resp.json()
|
||||
answer = RTCSessionDescription(sdp=answer_data["sdp"], type=answer_data["type"])
|
||||
await pc.setRemoteDescription(answer)
|
||||
|
||||
# 5. Wait for audio playback to finish
|
||||
max_stream_wait = 60
|
||||
elapsed = 0
|
||||
while elapsed < max_stream_wait:
|
||||
if audio_track.readyState == "ended":
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
elapsed += 0.5
|
||||
|
||||
# 6. Send STOP command and close connection
|
||||
try:
|
||||
channel.send(json.dumps({"cmd": "STOP"}))
|
||||
await asyncio.sleep(1)
|
||||
except Exception:
|
||||
pass # Channel may not be open if track ended quickly
|
||||
|
||||
await pc.close()
|
||||
|
||||
# 7. Poll until post-processing pipeline completes
|
||||
data = await poll_transcript_status(
|
||||
api_client, transcript_id, target="ended", max_wait=300
|
||||
)
|
||||
|
||||
# 8. Assertions
|
||||
assert data["status"] == "ended"
|
||||
assert data.get("title") and len(data["title"]) > 0, "Title should be non-empty"
|
||||
assert (
|
||||
data.get("long_summary") and len(data["long_summary"]) > 0
|
||||
), "Long summary should be non-empty"
|
||||
assert (
|
||||
data.get("short_summary") and len(data["short_summary"]) > 0
|
||||
), "Short summary should be non-empty"
|
||||
|
||||
# Topics are served from a separate endpoint
|
||||
topics_resp = await api_client.get(f"/transcripts/{transcript_id}/topics")
|
||||
assert topics_resp.status_code == 200, f"Failed to get topics: {topics_resp.text}"
|
||||
topics = topics_resp.json()
|
||||
assert len(topics) >= 1, "Should have at least 1 topic"
|
||||
for topic in topics:
|
||||
assert topic.get("title"), "Each topic should have a title"
|
||||
assert topic.get("summary"), "Each topic should have a summary"
|
||||
|
||||
assert data.get("duration", 0) > 0, "Duration should be positive"
|
||||
129
server/tests/integration/test_multitrack_pipeline.py
Normal file
129
server/tests/integration/test_multitrack_pipeline.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
Integration test: Multitrack → DailyMultitrackPipeline → full processing.
|
||||
|
||||
Exercises: S3 upload → DB recording setup → process endpoint →
|
||||
Hatchet DiarizationPipeline → mock Daily API → whisper per-track transcription →
|
||||
diarization → mixdown → LLM summarization/topics → status "ended".
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
|
||||
# Must match Daily's filename format: {recording_start_ts}-{participant_uuid}-cam-audio-{track_start_ts}
|
||||
# These UUIDs must match mock_daily_server.py participant IDs
|
||||
PARTICIPANT_A_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
|
||||
PARTICIPANT_B_ID = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
|
||||
TRACK_KEYS = [
|
||||
f"1700000000000-{PARTICIPANT_A_ID}-cam-audio-1700000001000",
|
||||
f"1700000000000-{PARTICIPANT_B_ID}-cam-audio-1700000001000",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multitrack_pipeline_end_to_end(
|
||||
api_client,
|
||||
s3_client,
|
||||
db_engine,
|
||||
test_records_dir,
|
||||
bucket_name,
|
||||
poll_transcript_status,
|
||||
):
|
||||
"""Set up multitrack recording in S3/DB and verify the full pipeline completes."""
|
||||
# 1. Upload test audio as two separate tracks to Garage S3
|
||||
audio_path = test_records_dir / "test_short.wav"
|
||||
assert audio_path.exists(), f"Test audio file not found: {audio_path}"
|
||||
|
||||
for track_key in TRACK_KEYS:
|
||||
s3_client.upload_file(
|
||||
str(audio_path),
|
||||
bucket_name,
|
||||
track_key,
|
||||
)
|
||||
|
||||
# 2. Create transcript via API
|
||||
resp = await api_client.post(
|
||||
"/transcripts",
|
||||
json={"name": "integration-multitrack-test"},
|
||||
)
|
||||
assert resp.status_code == 200, f"Failed to create transcript: {resp.text}"
|
||||
transcript = resp.json()
|
||||
transcript_id = transcript["id"]
|
||||
|
||||
# 3. Insert Recording row and link to transcript via direct DB access
|
||||
recording_id = f"rec-integration-{transcript_id[:8]}"
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
async with db_engine.begin() as conn:
|
||||
# Insert recording with track_keys
|
||||
await conn.execute(
|
||||
text("""
|
||||
INSERT INTO recording (id, bucket_name, object_key, recorded_at, status, track_keys)
|
||||
VALUES (:id, :bucket_name, :object_key, :recorded_at, :status, CAST(:track_keys AS json))
|
||||
"""),
|
||||
{
|
||||
"id": recording_id,
|
||||
"bucket_name": bucket_name,
|
||||
"object_key": TRACK_KEYS[0],
|
||||
"recorded_at": now,
|
||||
"status": "completed",
|
||||
"track_keys": json.dumps(TRACK_KEYS),
|
||||
},
|
||||
)
|
||||
|
||||
# Link recording to transcript and set status to uploaded
|
||||
await conn.execute(
|
||||
text("""
|
||||
UPDATE transcript
|
||||
SET recording_id = :recording_id, status = 'uploaded'
|
||||
WHERE id = :transcript_id
|
||||
"""),
|
||||
{
|
||||
"recording_id": recording_id,
|
||||
"transcript_id": transcript_id,
|
||||
},
|
||||
)
|
||||
|
||||
# 4. Trigger processing via process endpoint
|
||||
resp = await api_client.post(f"/transcripts/{transcript_id}/process")
|
||||
assert resp.status_code == 200, f"Process trigger failed: {resp.text}"
|
||||
|
||||
# 5. Poll until pipeline completes
|
||||
# The pipeline will call mock-daily for get_recording and get_participants
|
||||
# Accept "error" too — non-critical steps like action_items may fail due to
|
||||
# LLM parsing flakiness while core results (transcript, summaries) still exist.
|
||||
data = await poll_transcript_status(
|
||||
api_client, transcript_id, target=("ended", "error"), max_wait=300
|
||||
)
|
||||
|
||||
# 6. Assertions — verify core pipeline results regardless of final status
|
||||
assert data.get("title") and len(data["title"]) > 0, "Title should be non-empty"
|
||||
assert (
|
||||
data.get("long_summary") and len(data["long_summary"]) > 0
|
||||
), "Long summary should be non-empty"
|
||||
assert (
|
||||
data.get("short_summary") and len(data["short_summary"]) > 0
|
||||
), "Short summary should be non-empty"
|
||||
|
||||
# Topics are served from a separate endpoint
|
||||
topics_resp = await api_client.get(f"/transcripts/{transcript_id}/topics")
|
||||
assert topics_resp.status_code == 200, f"Failed to get topics: {topics_resp.text}"
|
||||
topics = topics_resp.json()
|
||||
assert len(topics) >= 1, "Should have at least 1 topic"
|
||||
for topic in topics:
|
||||
assert topic.get("title"), "Each topic should have a title"
|
||||
assert topic.get("summary"), "Each topic should have a summary"
|
||||
|
||||
# Participants are served from a separate endpoint
|
||||
participants_resp = await api_client.get(
|
||||
f"/transcripts/{transcript_id}/participants"
|
||||
)
|
||||
assert (
|
||||
participants_resp.status_code == 200
|
||||
), f"Failed to get participants: {participants_resp.text}"
|
||||
participants = participants_resp.json()
|
||||
assert (
|
||||
len(participants) >= 2
|
||||
), f"Expected at least 2 speakers for multitrack, got {len(participants)}"
|
||||
17
server/tests/test_app.py
Normal file
17
server/tests/test_app.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""Tests for app-level endpoints (root, not under /v1)."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint_returns_healthy():
|
||||
"""GET /health returns 200 and {"status": "healthy"} for probes and CI."""
|
||||
from httpx import AsyncClient
|
||||
|
||||
from reflector.app import app
|
||||
|
||||
# Health is at app root, not under /v1
|
||||
async with AsyncClient(app=app, base_url="http://test") as root_client:
|
||||
response = await root_client.get("/health")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"status": "healthy"}
|
||||
@@ -37,18 +37,3 @@ async def test_hatchet_client_can_replay_handles_exception():
|
||||
|
||||
# Should return False on error (workflow might be gone)
|
||||
assert can_replay is False
|
||||
|
||||
|
||||
def test_hatchet_client_raises_without_token():
|
||||
"""Test that get_client raises ValueError without token.
|
||||
|
||||
Useful: Catches if someone removes the token validation,
|
||||
which would cause cryptic errors later.
|
||||
"""
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
|
||||
with patch("reflector.hatchet.client.settings") as mock_settings:
|
||||
mock_settings.HATCHET_CLIENT_TOKEN = None
|
||||
|
||||
with pytest.raises(ValueError, match="HATCHET_CLIENT_TOKEN must be set"):
|
||||
HatchetClientManager.get_client()
|
||||
|
||||
233
server/tests/test_hatchet_file_pipeline.py
Normal file
233
server/tests/test_hatchet_file_pipeline.py
Normal file
@@ -0,0 +1,233 @@
|
||||
"""
|
||||
Tests for the FilePipeline Hatchet workflow.
|
||||
|
||||
Tests verify:
|
||||
1. with_error_handling behavior for file pipeline input model
|
||||
2. on_workflow_failure logic (don't overwrite 'ended' status)
|
||||
3. Input model validation
|
||||
"""
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from hatchet_sdk import NonRetryableException
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _noop_db_context():
|
||||
"""Async context manager that yields without touching the DB."""
|
||||
yield None
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def file_pipeline_module():
|
||||
"""Import file_pipeline with Hatchet client mocked."""
|
||||
mock_client = MagicMock()
|
||||
mock_client.workflow.return_value = MagicMock()
|
||||
with patch(
|
||||
"reflector.hatchet.client.HatchetClientManager.get_client",
|
||||
return_value=mock_client,
|
||||
):
|
||||
from reflector.hatchet.workflows import file_pipeline
|
||||
|
||||
return file_pipeline
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_file_input():
|
||||
"""Minimal FilePipelineInput for tests."""
|
||||
from reflector.hatchet.workflows.file_pipeline import FilePipelineInput
|
||||
|
||||
return FilePipelineInput(
|
||||
transcript_id="ts-file-123",
|
||||
room_id="room-456",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
"""Minimal Context-like object."""
|
||||
ctx = MagicMock()
|
||||
ctx.log = MagicMock()
|
||||
return ctx
|
||||
|
||||
|
||||
def test_file_pipeline_input_model():
|
||||
"""Test FilePipelineInput validation."""
|
||||
from reflector.hatchet.workflows.file_pipeline import FilePipelineInput
|
||||
|
||||
# Valid input with room_id
|
||||
input_with_room = FilePipelineInput(transcript_id="ts-123", room_id="room-456")
|
||||
assert input_with_room.transcript_id == "ts-123"
|
||||
assert input_with_room.room_id == "room-456"
|
||||
|
||||
# Valid input without room_id
|
||||
input_no_room = FilePipelineInput(transcript_id="ts-123")
|
||||
assert input_no_room.room_id is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_pipeline_error_handling_transient(
|
||||
file_pipeline_module, mock_file_input, mock_ctx
|
||||
):
|
||||
"""Transient exception must NOT set error status."""
|
||||
from reflector.hatchet.workflows.daily_multitrack_pipeline import (
|
||||
TaskName,
|
||||
with_error_handling,
|
||||
)
|
||||
|
||||
async def failing_task(input, ctx):
|
||||
raise httpx.TimeoutException("timed out")
|
||||
|
||||
wrapped = with_error_handling(TaskName.EXTRACT_AUDIO)(failing_task)
|
||||
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.daily_multitrack_pipeline.set_workflow_error_status",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_set_error:
|
||||
with pytest.raises(httpx.TimeoutException):
|
||||
await wrapped(mock_file_input, mock_ctx)
|
||||
|
||||
mock_set_error.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_pipeline_error_handling_hard_fail(
|
||||
file_pipeline_module, mock_file_input, mock_ctx
|
||||
):
|
||||
"""Hard-fail (ValueError) must set error status and raise NonRetryableException."""
|
||||
from reflector.hatchet.workflows.daily_multitrack_pipeline import (
|
||||
TaskName,
|
||||
with_error_handling,
|
||||
)
|
||||
|
||||
async def failing_task(input, ctx):
|
||||
raise ValueError("No audio file found")
|
||||
|
||||
wrapped = with_error_handling(TaskName.EXTRACT_AUDIO)(failing_task)
|
||||
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.daily_multitrack_pipeline.set_workflow_error_status",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_set_error:
|
||||
with pytest.raises(NonRetryableException) as exc_info:
|
||||
await wrapped(mock_file_input, mock_ctx)
|
||||
|
||||
assert "No audio file found" in str(exc_info.value)
|
||||
mock_set_error.assert_called_once_with("ts-file-123")
|
||||
|
||||
|
||||
def test_diarize_result_uses_plain_dicts():
|
||||
"""DiarizationSegment is a TypedDict (plain dict), not a Pydantic model.
|
||||
|
||||
The diarize task must serialize segments as plain dicts (not call .model_dump()),
|
||||
and assemble_transcript must be able to reconstruct them with DiarizationSegment(**s).
|
||||
This was a real bug: 'dict' object has no attribute 'model_dump'.
|
||||
"""
|
||||
from reflector.hatchet.workflows.file_pipeline import DiarizeResult
|
||||
from reflector.processors.types import DiarizationSegment
|
||||
|
||||
# DiarizationSegment is a TypedDict — instances are plain dicts
|
||||
segments = [
|
||||
DiarizationSegment(start=0.0, end=1.5, speaker=0),
|
||||
DiarizationSegment(start=1.5, end=3.0, speaker=1),
|
||||
]
|
||||
assert isinstance(segments[0], dict), "DiarizationSegment should be a plain dict"
|
||||
|
||||
# DiarizeResult should accept list[dict] directly (no model_dump needed)
|
||||
result = DiarizeResult(diarization=segments)
|
||||
assert result.diarization is not None
|
||||
assert len(result.diarization) == 2
|
||||
|
||||
# Consumer (assemble_transcript) reconstructs via DiarizationSegment(**s)
|
||||
reconstructed = [DiarizationSegment(**s) for s in result.diarization]
|
||||
assert reconstructed[0]["start"] == 0.0
|
||||
assert reconstructed[0]["speaker"] == 0
|
||||
assert reconstructed[1]["end"] == 3.0
|
||||
assert reconstructed[1]["speaker"] == 1
|
||||
|
||||
|
||||
def test_diarize_result_handles_none():
|
||||
"""DiarizeResult with no diarization data (diarization disabled)."""
|
||||
from reflector.hatchet.workflows.file_pipeline import DiarizeResult
|
||||
|
||||
result = DiarizeResult(diarization=None)
|
||||
assert result.diarization is None
|
||||
|
||||
result_default = DiarizeResult()
|
||||
assert result_default.diarization is None
|
||||
|
||||
|
||||
def test_transcribe_result_words_are_pydantic():
|
||||
"""TranscribeResult words come from Pydantic Word.model_dump() — verify roundtrip."""
|
||||
from reflector.hatchet.workflows.file_pipeline import TranscribeResult
|
||||
from reflector.processors.types import Word
|
||||
|
||||
words = [
|
||||
Word(text="hello", start=0.0, end=0.5),
|
||||
Word(text="world", start=0.5, end=1.0),
|
||||
]
|
||||
# Words are Pydantic models, so model_dump() works
|
||||
word_dicts = [w.model_dump() for w in words]
|
||||
result = TranscribeResult(words=word_dicts)
|
||||
|
||||
# Consumer reconstructs via Word(**w)
|
||||
reconstructed = [Word(**w) for w in result.words]
|
||||
assert reconstructed[0].text == "hello"
|
||||
assert reconstructed[1].start == 0.5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_pipeline_on_failure_sets_error_status(
|
||||
file_pipeline_module, mock_file_input, mock_ctx
|
||||
):
|
||||
"""on_workflow_failure sets error status when transcript is processing."""
|
||||
from reflector.hatchet.workflows.file_pipeline import on_workflow_failure
|
||||
|
||||
transcript_processing = MagicMock()
|
||||
transcript_processing.status = "processing"
|
||||
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.file_pipeline.fresh_db_connection",
|
||||
_noop_db_context,
|
||||
):
|
||||
with patch(
|
||||
"reflector.db.transcripts.transcripts_controller.get_by_id",
|
||||
new_callable=AsyncMock,
|
||||
return_value=transcript_processing,
|
||||
):
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.file_pipeline.set_workflow_error_status",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_set_error:
|
||||
await on_workflow_failure(mock_file_input, mock_ctx)
|
||||
mock_set_error.assert_called_once_with(mock_file_input.transcript_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_pipeline_on_failure_does_not_overwrite_ended(
|
||||
file_pipeline_module, mock_file_input, mock_ctx
|
||||
):
|
||||
"""on_workflow_failure must NOT overwrite 'ended' status."""
|
||||
from reflector.hatchet.workflows.file_pipeline import on_workflow_failure
|
||||
|
||||
transcript_ended = MagicMock()
|
||||
transcript_ended.status = "ended"
|
||||
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.file_pipeline.fresh_db_connection",
|
||||
_noop_db_context,
|
||||
):
|
||||
with patch(
|
||||
"reflector.db.transcripts.transcripts_controller.get_by_id",
|
||||
new_callable=AsyncMock,
|
||||
return_value=transcript_ended,
|
||||
):
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.file_pipeline.set_workflow_error_status",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_set_error:
|
||||
await on_workflow_failure(mock_file_input, mock_ctx)
|
||||
mock_set_error.assert_not_called()
|
||||
218
server/tests/test_hatchet_live_post_pipeline.py
Normal file
218
server/tests/test_hatchet_live_post_pipeline.py
Normal file
@@ -0,0 +1,218 @@
|
||||
"""
|
||||
Tests for the LivePostProcessingPipeline Hatchet workflow.
|
||||
|
||||
Tests verify:
|
||||
1. with_error_handling behavior for live post pipeline input model
|
||||
2. on_workflow_failure logic (don't overwrite 'ended' status)
|
||||
3. Input model validation
|
||||
4. pipeline_post() now triggers Hatchet instead of Celery chord
|
||||
"""
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from hatchet_sdk import NonRetryableException
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _noop_db_context():
|
||||
"""Async context manager that yields without touching the DB."""
|
||||
yield None
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def live_pipeline_module():
|
||||
"""Import live_post_pipeline with Hatchet client mocked."""
|
||||
mock_client = MagicMock()
|
||||
mock_client.workflow.return_value = MagicMock()
|
||||
with patch(
|
||||
"reflector.hatchet.client.HatchetClientManager.get_client",
|
||||
return_value=mock_client,
|
||||
):
|
||||
from reflector.hatchet.workflows import live_post_pipeline
|
||||
|
||||
return live_post_pipeline
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_live_input():
|
||||
"""Minimal LivePostPipelineInput for tests."""
|
||||
from reflector.hatchet.workflows.live_post_pipeline import LivePostPipelineInput
|
||||
|
||||
return LivePostPipelineInput(
|
||||
transcript_id="ts-live-789",
|
||||
room_id="room-abc",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
"""Minimal Context-like object."""
|
||||
ctx = MagicMock()
|
||||
ctx.log = MagicMock()
|
||||
return ctx
|
||||
|
||||
|
||||
def test_live_post_pipeline_input_model():
|
||||
"""Test LivePostPipelineInput validation."""
|
||||
from reflector.hatchet.workflows.live_post_pipeline import LivePostPipelineInput
|
||||
|
||||
# Valid input with room_id
|
||||
input_with_room = LivePostPipelineInput(transcript_id="ts-123", room_id="room-456")
|
||||
assert input_with_room.transcript_id == "ts-123"
|
||||
assert input_with_room.room_id == "room-456"
|
||||
|
||||
# Valid input without room_id
|
||||
input_no_room = LivePostPipelineInput(transcript_id="ts-123")
|
||||
assert input_no_room.room_id is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_pipeline_error_handling_transient(
|
||||
live_pipeline_module, mock_live_input, mock_ctx
|
||||
):
|
||||
"""Transient exception must NOT set error status."""
|
||||
from reflector.hatchet.workflows.daily_multitrack_pipeline import (
|
||||
TaskName,
|
||||
with_error_handling,
|
||||
)
|
||||
|
||||
async def failing_task(input, ctx):
|
||||
raise httpx.TimeoutException("timed out")
|
||||
|
||||
wrapped = with_error_handling(TaskName.WAVEFORM)(failing_task)
|
||||
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.daily_multitrack_pipeline.set_workflow_error_status",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_set_error:
|
||||
with pytest.raises(httpx.TimeoutException):
|
||||
await wrapped(mock_live_input, mock_ctx)
|
||||
|
||||
mock_set_error.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_pipeline_error_handling_hard_fail(
|
||||
live_pipeline_module, mock_live_input, mock_ctx
|
||||
):
|
||||
"""Hard-fail must set error status and raise NonRetryableException."""
|
||||
from reflector.hatchet.workflows.daily_multitrack_pipeline import (
|
||||
TaskName,
|
||||
with_error_handling,
|
||||
)
|
||||
|
||||
async def failing_task(input, ctx):
|
||||
raise ValueError("Transcript not found")
|
||||
|
||||
wrapped = with_error_handling(TaskName.WAVEFORM)(failing_task)
|
||||
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.daily_multitrack_pipeline.set_workflow_error_status",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_set_error:
|
||||
with pytest.raises(NonRetryableException) as exc_info:
|
||||
await wrapped(mock_live_input, mock_ctx)
|
||||
|
||||
assert "Transcript not found" in str(exc_info.value)
|
||||
mock_set_error.assert_called_once_with("ts-live-789")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_pipeline_on_failure_sets_error_status(
|
||||
live_pipeline_module, mock_live_input, mock_ctx
|
||||
):
|
||||
"""on_workflow_failure sets error status when transcript is processing."""
|
||||
from reflector.hatchet.workflows.live_post_pipeline import on_workflow_failure
|
||||
|
||||
transcript_processing = MagicMock()
|
||||
transcript_processing.status = "processing"
|
||||
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.live_post_pipeline.fresh_db_connection",
|
||||
_noop_db_context,
|
||||
):
|
||||
with patch(
|
||||
"reflector.db.transcripts.transcripts_controller.get_by_id",
|
||||
new_callable=AsyncMock,
|
||||
return_value=transcript_processing,
|
||||
):
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.live_post_pipeline.set_workflow_error_status",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_set_error:
|
||||
await on_workflow_failure(mock_live_input, mock_ctx)
|
||||
mock_set_error.assert_called_once_with(mock_live_input.transcript_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_pipeline_on_failure_does_not_overwrite_ended(
|
||||
live_pipeline_module, mock_live_input, mock_ctx
|
||||
):
|
||||
"""on_workflow_failure must NOT overwrite 'ended' status."""
|
||||
from reflector.hatchet.workflows.live_post_pipeline import on_workflow_failure
|
||||
|
||||
transcript_ended = MagicMock()
|
||||
transcript_ended.status = "ended"
|
||||
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.live_post_pipeline.fresh_db_connection",
|
||||
_noop_db_context,
|
||||
):
|
||||
with patch(
|
||||
"reflector.db.transcripts.transcripts_controller.get_by_id",
|
||||
new_callable=AsyncMock,
|
||||
return_value=transcript_ended,
|
||||
):
|
||||
with patch(
|
||||
"reflector.hatchet.workflows.live_post_pipeline.set_workflow_error_status",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_set_error:
|
||||
await on_workflow_failure(mock_live_input, mock_ctx)
|
||||
mock_set_error.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_post_triggers_hatchet():
|
||||
"""pipeline_post() should trigger Hatchet LivePostProcessingPipeline workflow."""
|
||||
with patch(
|
||||
"reflector.hatchet.client.HatchetClientManager.start_workflow",
|
||||
new_callable=AsyncMock,
|
||||
return_value="workflow-run-id",
|
||||
) as mock_start:
|
||||
from reflector.pipelines.main_live_pipeline import pipeline_post
|
||||
|
||||
await pipeline_post(transcript_id="ts-test-123", room_id="room-test")
|
||||
|
||||
mock_start.assert_called_once_with(
|
||||
"LivePostProcessingPipeline",
|
||||
{
|
||||
"transcript_id": "ts-test-123",
|
||||
"room_id": "room-test",
|
||||
},
|
||||
additional_metadata={"transcript_id": "ts-test-123"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_post_triggers_hatchet_without_room_id():
|
||||
"""pipeline_post() should handle None room_id."""
|
||||
with patch(
|
||||
"reflector.hatchet.client.HatchetClientManager.start_workflow",
|
||||
new_callable=AsyncMock,
|
||||
return_value="workflow-run-id",
|
||||
) as mock_start:
|
||||
from reflector.pipelines.main_live_pipeline import pipeline_post
|
||||
|
||||
await pipeline_post(transcript_id="ts-test-456")
|
||||
|
||||
mock_start.assert_called_once_with(
|
||||
"LivePostProcessingPipeline",
|
||||
{
|
||||
"transcript_id": "ts-test-456",
|
||||
"room_id": None,
|
||||
},
|
||||
additional_metadata={"transcript_id": "ts-test-456"},
|
||||
)
|
||||
90
server/tests/test_hatchet_trigger_migration.py
Normal file
90
server/tests/test_hatchet_trigger_migration.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Tests verifying Celery-to-Hatchet trigger migration.
|
||||
|
||||
Ensures that:
|
||||
1. process_recording triggers FilePipeline via Hatchet (not Celery)
|
||||
2. transcript_record_upload triggers FilePipeline via Hatchet (not Celery)
|
||||
3. Old Celery task references are no longer in active call sites
|
||||
"""
|
||||
|
||||
|
||||
def test_process_recording_does_not_import_celery_file_task():
|
||||
"""Verify process.py no longer imports task_pipeline_file_process."""
|
||||
import inspect
|
||||
|
||||
from reflector.worker import process
|
||||
|
||||
source = inspect.getsource(process)
|
||||
# Should not contain the old Celery task import
|
||||
assert "task_pipeline_file_process" not in source
|
||||
|
||||
|
||||
def test_transcripts_upload_does_not_import_celery_file_task():
|
||||
"""Verify transcripts_upload.py no longer imports task_pipeline_file_process."""
|
||||
import inspect
|
||||
|
||||
from reflector.views import transcripts_upload
|
||||
|
||||
source = inspect.getsource(transcripts_upload)
|
||||
# Should not contain the old Celery task import
|
||||
assert "task_pipeline_file_process" not in source
|
||||
|
||||
|
||||
def test_transcripts_upload_imports_hatchet():
|
||||
"""Verify transcripts_upload.py imports HatchetClientManager."""
|
||||
import inspect
|
||||
|
||||
from reflector.views import transcripts_upload
|
||||
|
||||
source = inspect.getsource(transcripts_upload)
|
||||
assert "HatchetClientManager" in source
|
||||
|
||||
|
||||
def test_pipeline_post_is_async():
|
||||
"""Verify pipeline_post is now async (Hatchet trigger)."""
|
||||
import asyncio
|
||||
|
||||
from reflector.pipelines.main_live_pipeline import pipeline_post
|
||||
|
||||
assert asyncio.iscoroutinefunction(pipeline_post)
|
||||
|
||||
|
||||
def test_transcript_process_service_does_not_import_celery_file_task():
|
||||
"""Verify transcript_process.py service no longer imports task_pipeline_file_process."""
|
||||
import inspect
|
||||
|
||||
from reflector.services import transcript_process
|
||||
|
||||
source = inspect.getsource(transcript_process)
|
||||
assert "task_pipeline_file_process" not in source
|
||||
|
||||
|
||||
def test_transcript_process_service_dispatch_uses_hatchet():
|
||||
"""Verify dispatch_transcript_processing uses HatchetClientManager for file processing."""
|
||||
import inspect
|
||||
|
||||
from reflector.services import transcript_process
|
||||
|
||||
source = inspect.getsource(transcript_process.dispatch_transcript_processing)
|
||||
assert "HatchetClientManager" in source
|
||||
assert "FilePipeline" in source
|
||||
|
||||
|
||||
def test_new_task_names_exist():
|
||||
"""Verify new TaskName constants were added for file and live pipelines."""
|
||||
from reflector.hatchet.constants import TaskName
|
||||
|
||||
# File pipeline tasks
|
||||
assert TaskName.EXTRACT_AUDIO == "extract_audio"
|
||||
assert TaskName.UPLOAD_AUDIO == "upload_audio"
|
||||
assert TaskName.TRANSCRIBE == "transcribe"
|
||||
assert TaskName.DIARIZE == "diarize"
|
||||
assert TaskName.ASSEMBLE_TRANSCRIPT == "assemble_transcript"
|
||||
assert TaskName.GENERATE_SUMMARIES == "generate_summaries"
|
||||
|
||||
# Live post-processing pipeline tasks
|
||||
assert TaskName.WAVEFORM == "waveform"
|
||||
assert TaskName.CONVERT_MP3 == "convert_mp3"
|
||||
assert TaskName.UPLOAD_MP3 == "upload_mp3"
|
||||
assert TaskName.REMOVE_UPLOAD == "remove_upload"
|
||||
assert TaskName.FINAL_SUMMARIES == "final_summaries"
|
||||
@@ -5,6 +5,7 @@ This test verifies the complete file processing pipeline without mocking much,
|
||||
ensuring all processors are correctly invoked and the happy path works correctly.
|
||||
"""
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from uuid import uuid4
|
||||
@@ -651,3 +652,43 @@ async def test_pipeline_file_process_no_audio_file(
|
||||
# This should fail when trying to open the file with av
|
||||
with pytest.raises(Exception):
|
||||
await pipeline.process(non_existent_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_on_title_does_not_overwrite_user_set_title():
|
||||
"""When transcript already has a title, on_title does not call update."""
|
||||
from reflector.db.transcripts import Transcript, TranscriptFinalTitle
|
||||
from reflector.pipelines.main_file_pipeline import PipelineMainFile
|
||||
|
||||
transcript_id = str(uuid4())
|
||||
transcript_with_title = Transcript(
|
||||
id=transcript_id,
|
||||
name="test",
|
||||
source_kind="file",
|
||||
title="User set title",
|
||||
)
|
||||
|
||||
controller = "reflector.pipelines.main_live_pipeline.transcripts_controller"
|
||||
with patch(f"{controller}.get_by_id", new_callable=AsyncMock) as mock_get:
|
||||
with patch(f"{controller}.update", new_callable=AsyncMock) as mock_update:
|
||||
with patch(
|
||||
f"{controller}.append_event", new_callable=AsyncMock
|
||||
) as mock_append:
|
||||
with patch(f"{controller}.transaction") as mock_txn:
|
||||
mock_get.return_value = transcript_with_title
|
||||
mock_append.return_value = None
|
||||
|
||||
@asynccontextmanager
|
||||
async def noop_txn():
|
||||
yield
|
||||
|
||||
mock_txn.return_value = noop_txn()
|
||||
|
||||
pipeline = PipelineMainFile(transcript_id=transcript_id)
|
||||
await pipeline.on_title(
|
||||
TranscriptFinalTitle(title="Generated title")
|
||||
)
|
||||
|
||||
mock_get.assert_called_once()
|
||||
mock_update.assert_not_called()
|
||||
mock_append.assert_called_once()
|
||||
|
||||
@@ -5,6 +5,7 @@ import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from conftest import authenticated_client_ctx
|
||||
from httpx_ws import aconnect_ws
|
||||
from uvicorn import Config, Server
|
||||
|
||||
@@ -382,3 +383,607 @@ async def test_audio_mp3_requires_token_for_owned_transcript(
|
||||
)
|
||||
resp2 = await client.get(f"/transcripts/{t.id}/audio/mp3", params={"token": token})
|
||||
assert resp2.status_code == 200
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Auth guards: anonymous blocked when PUBLIC_MODE=False
|
||||
# ======================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_cannot_create_transcript_when_not_public(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
resp = await client.post("/transcripts", json={"name": "anon-test"})
|
||||
assert resp.status_code == 401, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_cannot_process_transcript_when_not_public(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="process-test",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="public",
|
||||
)
|
||||
|
||||
resp = await client.post(f"/transcripts/{t.id}/process")
|
||||
assert resp.status_code == 401, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_cannot_upload_when_not_public(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="upload-test",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="public",
|
||||
)
|
||||
|
||||
# Minimal multipart upload
|
||||
resp = await client.post(
|
||||
f"/transcripts/{t.id}/record/upload",
|
||||
params={"chunk_number": 0, "total_chunks": 1},
|
||||
files={"chunk": ("test.mp3", b"fake-audio", "audio/mpeg")},
|
||||
)
|
||||
assert resp.status_code == 401, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_cannot_webrtc_record_when_not_public(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="webrtc-test",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="public",
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
f"/transcripts/{t.id}/record/webrtc",
|
||||
json={"sdp": "v=0\r\n", "type": "offer"},
|
||||
)
|
||||
assert resp.status_code == 401, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_cannot_start_meeting_recording_when_not_public(
|
||||
client, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
room = await rooms_controller.add(
|
||||
name="recording-auth-test",
|
||||
user_id="owner-rec",
|
||||
zulip_auto_post=False,
|
||||
zulip_stream="",
|
||||
zulip_topic="",
|
||||
is_locked=False,
|
||||
room_mode="normal",
|
||||
recording_type="cloud",
|
||||
recording_trigger="automatic-2nd-participant",
|
||||
is_shared=True,
|
||||
webhook_url="",
|
||||
webhook_secret="",
|
||||
)
|
||||
|
||||
meeting = await meetings_controller.create(
|
||||
id="meeting-rec-test",
|
||||
room_name="recording-auth-test",
|
||||
room_url="room-url",
|
||||
host_room_url="host-url",
|
||||
start_date=Room.model_fields["created_at"].default_factory(),
|
||||
end_date=Room.model_fields["created_at"].default_factory(),
|
||||
room=room,
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
f"/meetings/{meeting.id}/recordings/start",
|
||||
json={"type": "cloud", "instanceId": "00000000-0000-0000-0000-000000000001"},
|
||||
)
|
||||
assert resp.status_code == 401, resp.text
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Public mode: anonymous IS allowed when PUBLIC_MODE=True
|
||||
# ======================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_can_create_transcript_when_public(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
resp = await client.post("/transcripts", json={"name": "anon-public-test"})
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_can_list_transcripts_when_public(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
resp = await client.get("/transcripts")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_can_read_public_transcript(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="readable-test",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="public",
|
||||
)
|
||||
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_can_upload_when_public(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="upload-public-test",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="public",
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
f"/transcripts/{t.id}/record/upload",
|
||||
params={"chunk_number": 0, "total_chunks": 2},
|
||||
files={"chunk": ("test.mp3", b"fake-audio", "audio/mpeg")},
|
||||
)
|
||||
# Chunk 0 of 2 won't trigger av.open validation, so should succeed with "ok"
|
||||
# The key assertion: auth did NOT block us (no 401)
|
||||
assert resp.status_code != 401, f"Should not get 401 in public mode: {resp.text}"
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_can_start_meeting_recording_when_public(client, monkeypatch):
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
room = await rooms_controller.add(
|
||||
name="recording-public-test",
|
||||
user_id="owner-pub",
|
||||
zulip_auto_post=False,
|
||||
zulip_stream="",
|
||||
zulip_topic="",
|
||||
is_locked=False,
|
||||
room_mode="normal",
|
||||
recording_type="cloud",
|
||||
recording_trigger="automatic-2nd-participant",
|
||||
is_shared=True,
|
||||
webhook_url="",
|
||||
webhook_secret="",
|
||||
)
|
||||
|
||||
meeting = await meetings_controller.create(
|
||||
id="meeting-pub-test",
|
||||
room_name="recording-public-test",
|
||||
room_url="room-url",
|
||||
host_room_url="host-url",
|
||||
start_date=Room.model_fields["created_at"].default_factory(),
|
||||
end_date=Room.model_fields["created_at"].default_factory(),
|
||||
room=room,
|
||||
)
|
||||
|
||||
resp = await client.post(
|
||||
f"/meetings/{meeting.id}/recordings/start",
|
||||
json={"type": "cloud", "instanceId": "00000000-0000-0000-0000-000000000002"},
|
||||
)
|
||||
# Should not be 401 (may fail for other reasons like no Daily API, but auth passes)
|
||||
assert resp.status_code != 401, f"Should not get 401 in public mode: {resp.text}"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Authenticated user vs private data (own transcripts)
|
||||
# Authenticated owner should be able to create, read, and process
|
||||
# their own private transcripts even when PUBLIC_MODE=False
|
||||
# ======================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_can_create_transcript_private_mode(client, monkeypatch):
|
||||
"""Authenticated user can create transcripts even when PUBLIC_MODE=False."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.post("/transcripts", json={"name": "auth-private-create"})
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["user_id"] == "randomuserid"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_can_read_own_private_transcript(client, monkeypatch):
|
||||
"""Authenticated owner can read their own private transcript."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
# Create transcript owned by "randomuserid"
|
||||
t = await transcripts_controller.add(
|
||||
name="auth-private-read",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="randomuserid",
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_cannot_read_others_private_transcript(client, monkeypatch):
|
||||
"""Authenticated user cannot read another user's private transcript."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
# Create transcript owned by someone else
|
||||
t = await transcripts_controller.add(
|
||||
name="other-private",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="other-owner",
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 403, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_can_process_own_private_transcript(client, monkeypatch):
|
||||
"""Authenticated owner can trigger processing on their own private transcript."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="auth-private-process",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="randomuserid",
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.post(f"/transcripts/{t.id}/process")
|
||||
# Should pass auth (may fail for other reasons like validation, but not 401/403)
|
||||
assert resp.status_code not in (401, 403), resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_can_upload_to_own_private_transcript(client, monkeypatch):
|
||||
"""Authenticated owner can upload audio to their own private transcript."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="auth-private-upload",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="randomuserid",
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.post(
|
||||
f"/transcripts/{t.id}/record/upload",
|
||||
params={"chunk_number": 0, "total_chunks": 2},
|
||||
files={"chunk": ("test.mp3", b"fake-audio", "audio/mpeg")},
|
||||
)
|
||||
# Auth passes, chunk accepted (not final chunk so no av validation)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_can_webrtc_own_private_transcript(client, monkeypatch):
|
||||
"""Authenticated owner can start WebRTC recording on their own private transcript."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="auth-private-webrtc",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="randomuserid",
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.post(
|
||||
f"/transcripts/{t.id}/record/webrtc",
|
||||
json={"sdp": "v=0\r\n", "type": "offer"},
|
||||
)
|
||||
# Auth passes (may fail for other reasons like RTC setup, but not 401/403)
|
||||
assert resp.status_code not in (401, 403), resp.text
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Authenticated user vs semi-private data (other user's transcripts)
|
||||
# Any authenticated user should be able to READ semi-private transcripts
|
||||
# but NOT write to them (upload, process) since they don't own them
|
||||
# ======================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_can_read_others_semi_private_transcript(
|
||||
client, monkeypatch
|
||||
):
|
||||
"""Any authenticated user can read a semi-private transcript (link sharing)."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
# Create transcript owned by someone else with semi-private share mode
|
||||
t = await transcripts_controller.add(
|
||||
name="semi-private-readable",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="other-owner",
|
||||
share_mode="semi-private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_cannot_read_semi_private_transcript(client, monkeypatch):
|
||||
"""Anonymous user cannot read a semi-private transcript."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="semi-private-blocked",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="some-owner",
|
||||
share_mode="semi-private",
|
||||
)
|
||||
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 403, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_can_list_own_transcripts_private_mode(client, monkeypatch):
|
||||
"""Authenticated user can list their own transcripts when PUBLIC_MODE=False."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
await transcripts_controller.add(
|
||||
name="my-transcript",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="randomuserid",
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.get("/transcripts")
|
||||
assert resp.status_code == 200, resp.text
|
||||
items = resp.json()["items"]
|
||||
assert len(items) >= 1
|
||||
# All returned transcripts should belong to the user or be in shared rooms
|
||||
for item in items:
|
||||
assert item["user_id"] == "randomuserid" or item.get("room_id") is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_cannot_list_others_private_transcripts(
|
||||
client, monkeypatch
|
||||
):
|
||||
"""Authenticated user should NOT see another user's private transcripts in the list."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
await transcripts_controller.add(
|
||||
name="hidden-from-others",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="secret-owner",
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.get("/transcripts")
|
||||
assert resp.status_code == 200, resp.text
|
||||
items = resp.json()["items"]
|
||||
# Should not contain transcripts owned by "secret-owner"
|
||||
for item in items:
|
||||
assert (
|
||||
item.get("user_id") != "secret-owner"
|
||||
), f"Leaked private transcript: {item['id']}"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Anonymous-created transcripts (user_id=None)
|
||||
# These transcripts bypass share_mode checks entirely in get_by_id_for_http.
|
||||
# They should always be accessible to everyone regardless of PUBLIC_MODE
|
||||
# or share_mode setting, because there is no owner to restrict access.
|
||||
# ======================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_transcript_accessible_when_public_mode_true(
|
||||
client, monkeypatch
|
||||
):
|
||||
"""Anonymous transcript (user_id=None) is accessible even with default private share_mode
|
||||
when PUBLIC_MODE=True."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="anon-transcript-public-mode",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="private", # share_mode is irrelevant for user_id=None
|
||||
)
|
||||
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_transcript_accessible_when_public_mode_false(
|
||||
client, monkeypatch
|
||||
):
|
||||
"""Anonymous transcript (user_id=None) is accessible by authenticated users
|
||||
even when PUBLIC_MODE=False. The transcript has no owner, so share_mode is bypassed."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="anon-transcript-private-mode",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_transcript_accessible_regardless_of_share_mode(
|
||||
client, monkeypatch
|
||||
):
|
||||
"""Anonymous transcripts (user_id=None) are accessible regardless of share_mode value.
|
||||
Tests all three share modes to confirm the user_id=None bypass works consistently."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
for mode in ("private", "semi-private", "public"):
|
||||
t = await transcripts_controller.add(
|
||||
name=f"anon-share-{mode}",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode=mode,
|
||||
)
|
||||
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 200, f"Failed for share_mode={mode}: {resp.text}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_transcript_readable_by_different_authenticated_user(
|
||||
client, monkeypatch
|
||||
):
|
||||
"""An authenticated user can read anonymous transcripts (user_id=None) even with
|
||||
private share_mode, because the no-owner bypass applies."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="anon-read-by-auth-user",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["user_id"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_transcript_in_list_when_public_mode(client, monkeypatch):
|
||||
"""Anonymous transcripts appear in the transcript list when PUBLIC_MODE=True."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="anon-in-list",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
resp = await client.get("/transcripts")
|
||||
assert resp.status_code == 200, resp.text
|
||||
ids = [item["id"] for item in resp.json()["items"]]
|
||||
assert t.id in ids, "Anonymous transcript should appear in the public list"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_transcript_audio_accessible(client, monkeypatch, tmpdir):
|
||||
"""Anonymous transcript audio (mp3) is accessible without authentication
|
||||
because user_id=None bypasses share_mode checks."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
monkeypatch.setattr(settings, "DATA_DIR", Path(tmpdir).as_posix())
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="anon-audio-access",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id=None,
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
tr = await transcripts_controller.get_by_id(t.id)
|
||||
await transcripts_controller.update(tr, {"status": "ended"})
|
||||
|
||||
# Copy fixture audio to transcript path
|
||||
audio_path = Path(__file__).parent / "records" / "test_mathieu_hello.mp3"
|
||||
tr.audio_mp3_filename.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy(audio_path, tr.audio_mp3_filename)
|
||||
|
||||
resp = await client.get(f"/transcripts/{t.id}/audio/mp3")
|
||||
assert (
|
||||
resp.status_code == 200
|
||||
), f"Anonymous transcript audio should be accessible: {resp.text}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_owned_transcript_not_accessible_by_anon_when_not_public(
|
||||
client, monkeypatch
|
||||
):
|
||||
"""Contrast test: owned transcript with private share_mode is NOT accessible
|
||||
to anonymous users when PUBLIC_MODE=False. This confirms that the user_id=None
|
||||
bypass only applies to anonymous transcripts, not to all transcripts."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
t = await transcripts_controller.add(
|
||||
name="owned-private-contrast",
|
||||
source_kind=SourceKind.LIVE,
|
||||
user_id="some-owner",
|
||||
share_mode="private",
|
||||
)
|
||||
|
||||
resp = await client.get(f"/transcripts/{t.id}")
|
||||
assert (
|
||||
resp.status_code == 403
|
||||
), f"Owned private transcript should be denied to anonymous: {resp.text}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_authenticated_can_start_meeting_recording_private_mode(
|
||||
client, monkeypatch
|
||||
):
|
||||
"""Authenticated user can start recording in non-public mode."""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", False)
|
||||
|
||||
room = await rooms_controller.add(
|
||||
name="auth-recording-test",
|
||||
user_id="randomuserid",
|
||||
zulip_auto_post=False,
|
||||
zulip_stream="",
|
||||
zulip_topic="",
|
||||
is_locked=False,
|
||||
room_mode="normal",
|
||||
recording_type="cloud",
|
||||
recording_trigger="automatic-2nd-participant",
|
||||
is_shared=True,
|
||||
webhook_url="",
|
||||
webhook_secret="",
|
||||
)
|
||||
|
||||
meeting = await meetings_controller.create(
|
||||
id="meeting-auth-rec",
|
||||
room_name="auth-recording-test",
|
||||
room_url="room-url",
|
||||
host_room_url="host-url",
|
||||
start_date=Room.model_fields["created_at"].default_factory(),
|
||||
end_date=Room.model_fields["created_at"].default_factory(),
|
||||
room=room,
|
||||
)
|
||||
|
||||
async with authenticated_client_ctx():
|
||||
resp = await client.post(
|
||||
f"/meetings/{meeting.id}/recordings/start",
|
||||
json={
|
||||
"type": "cloud",
|
||||
"instanceId": "00000000-0000-0000-0000-000000000003",
|
||||
},
|
||||
)
|
||||
# Auth passes (may fail for Daily API reasons, but not 401)
|
||||
assert resp.status_code != 401, resp.text
|
||||
|
||||
@@ -340,8 +340,13 @@ async def test_transcript_formats_with_overlapping_speakers_multitrack():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_transcript_format_text(client):
|
||||
async def test_api_transcript_format_text(monkeypatch, client):
|
||||
"""Test GET /transcripts/{id} with transcript_format=text."""
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post("/transcripts", json={"name": "Test transcript"})
|
||||
assert response.status_code == 200
|
||||
tid = response.json()["id"]
|
||||
@@ -390,8 +395,13 @@ async def test_api_transcript_format_text(client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_transcript_format_text_timestamped(client):
|
||||
async def test_api_transcript_format_text_timestamped(monkeypatch, client):
|
||||
"""Test GET /transcripts/{id} with transcript_format=text-timestamped."""
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post("/transcripts", json={"name": "Test transcript"})
|
||||
assert response.status_code == 200
|
||||
tid = response.json()["id"]
|
||||
@@ -441,8 +451,13 @@ async def test_api_transcript_format_text_timestamped(client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_transcript_format_webvtt_named(client):
|
||||
async def test_api_transcript_format_webvtt_named(monkeypatch, client):
|
||||
"""Test GET /transcripts/{id} with transcript_format=webvtt-named."""
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post("/transcripts", json={"name": "Test transcript"})
|
||||
assert response.status_code == 200
|
||||
tid = response.json()["id"]
|
||||
@@ -491,8 +506,13 @@ async def test_api_transcript_format_webvtt_named(client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_transcript_format_json(client):
|
||||
async def test_api_transcript_format_json(monkeypatch, client):
|
||||
"""Test GET /transcripts/{id} with transcript_format=json."""
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post("/transcripts", json={"name": "Test transcript"})
|
||||
assert response.status_code == 200
|
||||
tid = response.json()["id"]
|
||||
@@ -544,8 +564,13 @@ async def test_api_transcript_format_json(client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_transcript_format_default_is_text(client):
|
||||
async def test_api_transcript_format_default_is_text(monkeypatch, client):
|
||||
"""Test GET /transcripts/{id} defaults to text format."""
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post("/transcripts", json={"name": "Test transcript"})
|
||||
assert response.status_code == 200
|
||||
tid = response.json()["id"]
|
||||
@@ -654,12 +679,18 @@ async def test_api_topics_endpoint_multitrack_segmentation(client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_topics_endpoint_non_multitrack_segmentation(client):
|
||||
async def test_api_topics_endpoint_non_multitrack_segmentation(monkeypatch, client):
|
||||
"""Test GET /transcripts/{id}/topics uses default segmentation for non-multitrack.
|
||||
|
||||
Ensures backward compatibility - transcripts without multitrack recordings
|
||||
should continue using the default speaker-change-based segmentation.
|
||||
"""
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
|
||||
from reflector.db.transcripts import (
|
||||
TranscriptParticipant,
|
||||
TranscriptTopic,
|
||||
|
||||
@@ -5,7 +5,12 @@ from reflector.db.transcripts import transcripts_controller
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_create(client):
|
||||
async def test_transcript_create(monkeypatch, client):
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post("/transcripts", json={"name": "test"})
|
||||
assert response.status_code == 200
|
||||
assert response.json()["name"] == "test"
|
||||
@@ -110,6 +115,33 @@ async def test_transcript_get_update_title(authenticated_client, client):
|
||||
assert response.json()["title"] == "test_title"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_status_emits_status_event_and_updates_transcript(
|
||||
monkeypatch, client
|
||||
):
|
||||
"""set_status adds a STATUS event and updates the transcript status (broadcast for WebSocket)."""
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post("/transcripts", json={"name": "Status test"})
|
||||
assert response.status_code == 200
|
||||
transcript_id = response.json()["id"]
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(transcript_id)
|
||||
assert transcript is not None
|
||||
assert transcript.status == "idle"
|
||||
|
||||
event = await transcripts_controller.set_status(transcript_id, "processing")
|
||||
assert event is not None
|
||||
assert event.event == "STATUS"
|
||||
assert event.data.get("value") == "processing"
|
||||
|
||||
updated = await transcripts_controller.get_by_id(transcript_id)
|
||||
assert updated.status == "processing"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcripts_list_anonymous(client):
|
||||
# XXX this test is a bit fragile, as it depends on the storage which
|
||||
@@ -233,3 +265,43 @@ async def test_transcript_get_returns_null_room_name_when_no_room(
|
||||
assert response.status_code == 200
|
||||
assert response.json()["room_id"] is None
|
||||
assert response.json()["room_name"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcripts_list_filtered_by_room_id(authenticated_client, client):
|
||||
"""GET /transcripts?room_id=X returns only transcripts for that room."""
|
||||
# Use same user as authenticated_client (conftest uses "randomuserid")
|
||||
user_id = "randomuserid"
|
||||
room = await rooms_controller.add(
|
||||
name="room-for-list-filter",
|
||||
user_id=user_id,
|
||||
zulip_auto_post=False,
|
||||
zulip_stream="",
|
||||
zulip_topic="",
|
||||
is_locked=False,
|
||||
room_mode="normal",
|
||||
recording_type="cloud",
|
||||
recording_trigger="automatic-2nd-participant",
|
||||
is_shared=False,
|
||||
webhook_url="",
|
||||
webhook_secret="",
|
||||
)
|
||||
in_room = await transcripts_controller.add(
|
||||
name="in-room",
|
||||
source_kind="file",
|
||||
room_id=room.id,
|
||||
user_id=user_id,
|
||||
)
|
||||
other = await transcripts_controller.add(
|
||||
name="no-room",
|
||||
source_kind="file",
|
||||
room_id=None,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
response = await client.get("/transcripts", params={"room_id": room.id})
|
||||
assert response.status_code == 200
|
||||
items = response.json()["items"]
|
||||
ids = [t["id"] for t in items]
|
||||
assert in_room.id in ids
|
||||
assert other.id not in ids
|
||||
|
||||
@@ -5,10 +5,13 @@ import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fake_transcript(tmpdir, client):
|
||||
async def fake_transcript(tmpdir, client, monkeypatch):
|
||||
from reflector.settings import settings
|
||||
from reflector.views.transcripts import transcripts_controller
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
settings.DATA_DIR = Path(tmpdir)
|
||||
|
||||
# create a transcript
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import time
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from reflector.settings import settings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def app_lifespan():
|
||||
@@ -25,8 +25,6 @@ async def client(app_lifespan):
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_database")
|
||||
@pytest.mark.usefixtures("celery_session_app")
|
||||
@pytest.mark.usefixtures("celery_session_worker")
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_process(
|
||||
tmpdir,
|
||||
@@ -36,7 +34,16 @@ async def test_transcript_process(
|
||||
dummy_file_diarization,
|
||||
dummy_storage,
|
||||
client,
|
||||
monkeypatch,
|
||||
mock_hatchet_client,
|
||||
):
|
||||
"""Test upload + process dispatch via Hatchet.
|
||||
|
||||
The file pipeline is now dispatched to Hatchet (fire-and-forget),
|
||||
so we verify the workflow was triggered rather than polling for completion.
|
||||
"""
|
||||
monkeypatch.setattr(settings, "PUBLIC_MODE", True)
|
||||
|
||||
# create a transcript
|
||||
response = await client.post("/transcripts", json={"name": "test"})
|
||||
assert response.status_code == 200
|
||||
@@ -57,61 +64,58 @@ async def test_transcript_process(
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
# wait for processing to finish (max 1 minute)
|
||||
timeout_seconds = 60
|
||||
start_time = time.monotonic()
|
||||
while (time.monotonic() - start_time) < timeout_seconds:
|
||||
# fetch the transcript and check if it is ended
|
||||
resp = await client.get(f"/transcripts/{tid}")
|
||||
assert resp.status_code == 200
|
||||
if resp.json()["status"] in ("ended", "error"):
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
else:
|
||||
pytest.fail(f"Initial processing timed out after {timeout_seconds} seconds")
|
||||
# Verify Hatchet workflow was dispatched (from upload endpoint)
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
|
||||
# restart the processing
|
||||
response = await client.post(
|
||||
f"/transcripts/{tid}/process",
|
||||
HatchetClientManager.start_workflow.assert_called_once_with(
|
||||
"FilePipeline",
|
||||
{"transcript_id": tid},
|
||||
additional_metadata={"transcript_id": tid},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# wait for processing to finish (max 1 minute)
|
||||
timeout_seconds = 60
|
||||
start_time = time.monotonic()
|
||||
while (time.monotonic() - start_time) < timeout_seconds:
|
||||
# fetch the transcript and check if it is ended
|
||||
resp = await client.get(f"/transcripts/{tid}")
|
||||
assert resp.status_code == 200
|
||||
if resp.json()["status"] in ("ended", "error"):
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
else:
|
||||
pytest.fail(f"Restart processing timed out after {timeout_seconds} seconds")
|
||||
# Verify transcript status was set to "uploaded"
|
||||
resp = await client.get(f"/transcripts/{tid}")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "uploaded"
|
||||
|
||||
# check the transcript is ended
|
||||
transcript = resp.json()
|
||||
assert transcript["status"] == "ended"
|
||||
assert transcript["short_summary"] == "LLM SHORT SUMMARY"
|
||||
assert transcript["title"] == "Llm Title"
|
||||
# Reset mock for reprocess test
|
||||
HatchetClientManager.start_workflow.reset_mock()
|
||||
|
||||
# check topics and transcript
|
||||
response = await client.get(f"/transcripts/{tid}/topics")
|
||||
assert response.status_code == 200
|
||||
assert len(response.json()) == 1
|
||||
assert "Hello world. How are you today?" in response.json()[0]["transcript"]
|
||||
# Clear workflow_run_id so /process endpoint can dispatch again
|
||||
from reflector.db.transcripts import transcripts_controller
|
||||
|
||||
transcript = await transcripts_controller.get_by_id(tid)
|
||||
await transcripts_controller.update(transcript, {"workflow_run_id": None})
|
||||
|
||||
# Reprocess via /process endpoint
|
||||
with patch(
|
||||
"reflector.services.transcript_process.task_is_scheduled_or_active",
|
||||
return_value=False,
|
||||
):
|
||||
response = await client.post(f"/transcripts/{tid}/process")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
# Verify second Hatchet dispatch (from /process endpoint)
|
||||
HatchetClientManager.start_workflow.assert_called_once()
|
||||
call_kwargs = HatchetClientManager.start_workflow.call_args.kwargs
|
||||
assert call_kwargs["workflow_name"] == "FilePipeline"
|
||||
assert call_kwargs["input_data"]["transcript_id"] == tid
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_database")
|
||||
@pytest.mark.asyncio
|
||||
async def test_whereby_recording_uses_file_pipeline(client):
|
||||
async def test_whereby_recording_uses_file_pipeline(monkeypatch, client):
|
||||
"""Test that Whereby recordings (bucket_name but no track_keys) use file pipeline"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from reflector.db.recordings import Recording, recordings_controller
|
||||
from reflector.db.transcripts import transcripts_controller
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
|
||||
# Create transcript with Whereby recording (has bucket_name, no track_keys)
|
||||
transcript = await transcripts_controller.add(
|
||||
@@ -139,31 +143,41 @@ async def test_whereby_recording_uses_file_pipeline(client):
|
||||
|
||||
with (
|
||||
patch(
|
||||
"reflector.services.transcript_process.task_pipeline_file_process"
|
||||
) as mock_file_pipeline,
|
||||
"reflector.services.transcript_process.task_is_scheduled_or_active",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"reflector.services.transcript_process.HatchetClientManager"
|
||||
) as mock_hatchet,
|
||||
):
|
||||
mock_hatchet.start_workflow = AsyncMock(return_value="test-workflow-id")
|
||||
|
||||
response = await client.post(f"/transcripts/{transcript.id}/process")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
# Whereby recordings should use file pipeline, not Hatchet
|
||||
mock_file_pipeline.delay.assert_called_once_with(transcript_id=transcript.id)
|
||||
mock_hatchet.start_workflow.assert_not_called()
|
||||
# Whereby recordings should use Hatchet FilePipeline
|
||||
mock_hatchet.start_workflow.assert_called_once()
|
||||
call_kwargs = mock_hatchet.start_workflow.call_args.kwargs
|
||||
assert call_kwargs["workflow_name"] == "FilePipeline"
|
||||
assert call_kwargs["input_data"]["transcript_id"] == transcript.id
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_database")
|
||||
@pytest.mark.asyncio
|
||||
async def test_dailyco_recording_uses_multitrack_pipeline(client):
|
||||
async def test_dailyco_recording_uses_multitrack_pipeline(monkeypatch, client):
|
||||
"""Test that Daily.co recordings (bucket_name + track_keys) use multitrack pipeline"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from reflector.db.recordings import Recording, recordings_controller
|
||||
from reflector.db.rooms import rooms_controller
|
||||
from reflector.db.transcripts import transcripts_controller
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
|
||||
room = await rooms_controller.add(
|
||||
name="test-room",
|
||||
@@ -208,8 +222,9 @@ async def test_dailyco_recording_uses_multitrack_pipeline(client):
|
||||
|
||||
with (
|
||||
patch(
|
||||
"reflector.services.transcript_process.task_pipeline_file_process"
|
||||
) as mock_file_pipeline,
|
||||
"reflector.services.transcript_process.task_is_scheduled_or_active",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"reflector.services.transcript_process.HatchetClientManager"
|
||||
) as mock_hatchet,
|
||||
@@ -221,7 +236,7 @@ async def test_dailyco_recording_uses_multitrack_pipeline(client):
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
# Daily.co multitrack recordings should use Hatchet workflow
|
||||
# Daily.co multitrack recordings should use Hatchet DiarizationPipeline
|
||||
mock_hatchet.start_workflow.assert_called_once()
|
||||
call_kwargs = mock_hatchet.start_workflow.call_args.kwargs
|
||||
assert call_kwargs["workflow_name"] == "DiarizationPipeline"
|
||||
@@ -230,18 +245,22 @@ async def test_dailyco_recording_uses_multitrack_pipeline(client):
|
||||
assert call_kwargs["input_data"]["tracks"] == [
|
||||
{"s3_key": k} for k in track_keys
|
||||
]
|
||||
mock_file_pipeline.delay.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_database")
|
||||
@pytest.mark.asyncio
|
||||
async def test_reprocess_error_transcript_passes_force(client):
|
||||
async def test_reprocess_error_transcript_passes_force(monkeypatch, client):
|
||||
"""When transcript status is 'error', reprocess passes force=True to start fresh workflow."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from reflector.db.recordings import Recording, recordings_controller
|
||||
from reflector.db.rooms import rooms_controller
|
||||
from reflector.db.transcripts import transcripts_controller
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
|
||||
room = await rooms_controller.add(
|
||||
name="test-room",
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
# FIXME test status of transcript
|
||||
# FIXME test websocket connection after RTC is finished still send the full events
|
||||
# FIXME try with locked session, RTC should not work
|
||||
# TODO: add integration tests for post-processing (LivePostPipeline) with a real
|
||||
# Hatchet instance. These tests currently only cover the live pipeline.
|
||||
# Post-processing events (WAVEFORM, FINAL_*, DURATION, STATUS=ended, mp3)
|
||||
# are now dispatched via Hatchet and tested in test_hatchet_live_post_pipeline.py.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
@@ -49,7 +53,7 @@ class ThreadedUvicorn:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def appserver(tmpdir, setup_database, celery_session_app, celery_session_worker):
|
||||
def appserver(tmpdir, setup_database):
|
||||
import threading
|
||||
|
||||
from reflector.app import app
|
||||
@@ -119,8 +123,6 @@ def appserver(tmpdir, setup_database, celery_session_app, celery_session_worker)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_database")
|
||||
@pytest.mark.usefixtures("celery_session_app")
|
||||
@pytest.mark.usefixtures("celery_session_worker")
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_rtc_and_websocket(
|
||||
tmpdir,
|
||||
@@ -133,10 +135,18 @@ async def test_transcript_rtc_and_websocket(
|
||||
fake_mp3_upload,
|
||||
appserver,
|
||||
client,
|
||||
monkeypatch,
|
||||
mock_hatchet_client,
|
||||
):
|
||||
# goal: start the server, exchange RTC, receive websocket events
|
||||
# because of that, we need to start the server in a thread
|
||||
# to be able to connect with aiortc
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
|
||||
server, host, port = appserver
|
||||
|
||||
# create a transcript
|
||||
@@ -201,35 +211,30 @@ async def test_transcript_rtc_and_websocket(
|
||||
stream_client.channel.send(json.dumps({"cmd": "STOP"}))
|
||||
await stream_client.stop()
|
||||
|
||||
# wait the processing to finish
|
||||
timeout = 120
|
||||
# Wait for live pipeline to flush (it dispatches post-processing to Hatchet)
|
||||
timeout = 30
|
||||
while True:
|
||||
# fetch the transcript and check if it is ended
|
||||
resp = await client.get(f"/transcripts/{tid}")
|
||||
assert resp.status_code == 200
|
||||
if resp.json()["status"] in ("ended", "error"):
|
||||
if resp.json()["status"] in ("processing", "ended", "error"):
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
timeout -= 1
|
||||
if timeout < 0:
|
||||
raise TimeoutError("Timeout while waiting for transcript to be ended")
|
||||
|
||||
if resp.json()["status"] != "ended":
|
||||
raise TimeoutError("Transcript processing failed")
|
||||
raise TimeoutError("Timeout waiting for live pipeline to finish")
|
||||
|
||||
# stop websocket task
|
||||
websocket_task.cancel()
|
||||
|
||||
# check events
|
||||
# check live pipeline events
|
||||
assert len(events) > 0
|
||||
from pprint import pprint
|
||||
|
||||
pprint(events)
|
||||
|
||||
# get events list
|
||||
eventnames = [e["event"] for e in events]
|
||||
|
||||
# check events
|
||||
# Live pipeline produces TRANSCRIPT and TOPIC events during RTC
|
||||
assert "TRANSCRIPT" in eventnames
|
||||
ev = events[eventnames.index("TRANSCRIPT")]
|
||||
assert ev["data"]["text"].startswith("Hello world.")
|
||||
@@ -242,50 +247,18 @@ async def test_transcript_rtc_and_websocket(
|
||||
assert ev["data"]["transcript"].startswith("Hello world.")
|
||||
assert ev["data"]["timestamp"] == 0.0
|
||||
|
||||
assert "FINAL_LONG_SUMMARY" in eventnames
|
||||
ev = events[eventnames.index("FINAL_LONG_SUMMARY")]
|
||||
assert ev["data"]["long_summary"] == "LLM LONG SUMMARY"
|
||||
|
||||
assert "FINAL_SHORT_SUMMARY" in eventnames
|
||||
ev = events[eventnames.index("FINAL_SHORT_SUMMARY")]
|
||||
assert ev["data"]["short_summary"] == "LLM SHORT SUMMARY"
|
||||
|
||||
assert "FINAL_TITLE" in eventnames
|
||||
ev = events[eventnames.index("FINAL_TITLE")]
|
||||
assert ev["data"]["title"] == "Llm Title"
|
||||
|
||||
assert "WAVEFORM" in eventnames
|
||||
ev = events[eventnames.index("WAVEFORM")]
|
||||
assert isinstance(ev["data"]["waveform"], list)
|
||||
assert len(ev["data"]["waveform"]) >= 250
|
||||
waveform_resp = await client.get(f"/transcripts/{tid}/audio/waveform")
|
||||
assert waveform_resp.status_code == 200
|
||||
assert waveform_resp.headers["content-type"] == "application/json"
|
||||
assert isinstance(waveform_resp.json()["data"], list)
|
||||
assert len(waveform_resp.json()["data"]) >= 250
|
||||
|
||||
# check status order
|
||||
# Live pipeline status progression
|
||||
statuses = [e["data"]["value"] for e in events if e["event"] == "STATUS"]
|
||||
assert "recording" in statuses
|
||||
assert "processing" in statuses
|
||||
assert statuses.index("recording") < statuses.index("processing")
|
||||
assert statuses.index("processing") < statuses.index("ended")
|
||||
|
||||
# ensure the last event received is ended
|
||||
assert events[-1]["event"] == "STATUS"
|
||||
assert events[-1]["data"]["value"] == "ended"
|
||||
|
||||
# check on the latest response that the audio duration is > 0
|
||||
assert resp.json()["duration"] > 0
|
||||
assert "DURATION" in eventnames
|
||||
|
||||
# check that audio/mp3 is available
|
||||
audio_resp = await client.get(f"/transcripts/{tid}/audio/mp3")
|
||||
assert audio_resp.status_code == 200
|
||||
assert audio_resp.headers["Content-Type"] == "audio/mpeg"
|
||||
# Post-processing (WAVEFORM, FINAL_*, DURATION, mp3, STATUS=ended) is now
|
||||
# dispatched to Hatchet via LivePostPipeline — not tested here.
|
||||
# See test_hatchet_live_post_pipeline.py for post-processing tests.
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_database")
|
||||
@pytest.mark.usefixtures("celery_session_app")
|
||||
@pytest.mark.usefixtures("celery_session_worker")
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_rtc_and_websocket_and_fr(
|
||||
tmpdir,
|
||||
@@ -298,11 +271,19 @@ async def test_transcript_rtc_and_websocket_and_fr(
|
||||
fake_mp3_upload,
|
||||
appserver,
|
||||
client,
|
||||
monkeypatch,
|
||||
mock_hatchet_client,
|
||||
):
|
||||
# goal: start the server, exchange RTC, receive websocket events
|
||||
# because of that, we need to start the server in a thread
|
||||
# to be able to connect with aiortc
|
||||
# with target french language
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
|
||||
server, host, port = appserver
|
||||
|
||||
# create a transcript
|
||||
@@ -368,42 +349,34 @@ async def test_transcript_rtc_and_websocket_and_fr(
|
||||
# instead of waiting a long time, we just send a STOP
|
||||
stream_client.channel.send(json.dumps({"cmd": "STOP"}))
|
||||
|
||||
# wait the processing to finish
|
||||
await asyncio.sleep(2)
|
||||
|
||||
await stream_client.stop()
|
||||
|
||||
# wait the processing to finish
|
||||
timeout = 120
|
||||
# Wait for live pipeline to flush
|
||||
timeout = 30
|
||||
while True:
|
||||
# fetch the transcript and check if it is ended
|
||||
resp = await client.get(f"/transcripts/{tid}")
|
||||
assert resp.status_code == 200
|
||||
if resp.json()["status"] == "ended":
|
||||
if resp.json()["status"] in ("processing", "ended", "error"):
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
timeout -= 1
|
||||
if timeout < 0:
|
||||
raise TimeoutError("Timeout while waiting for transcript to be ended")
|
||||
|
||||
if resp.json()["status"] != "ended":
|
||||
raise TimeoutError("Transcript processing failed")
|
||||
|
||||
await asyncio.sleep(2)
|
||||
raise TimeoutError("Timeout waiting for live pipeline to finish")
|
||||
|
||||
# stop websocket task
|
||||
websocket_task.cancel()
|
||||
|
||||
# check events
|
||||
# check live pipeline events
|
||||
assert len(events) > 0
|
||||
from pprint import pprint
|
||||
|
||||
pprint(events)
|
||||
|
||||
# get events list
|
||||
eventnames = [e["event"] for e in events]
|
||||
|
||||
# check events
|
||||
# Live pipeline produces TRANSCRIPT with translation
|
||||
assert "TRANSCRIPT" in eventnames
|
||||
ev = events[eventnames.index("TRANSCRIPT")]
|
||||
assert ev["data"]["text"].startswith("Hello world.")
|
||||
@@ -416,23 +389,11 @@ async def test_transcript_rtc_and_websocket_and_fr(
|
||||
assert ev["data"]["transcript"].startswith("Hello world.")
|
||||
assert ev["data"]["timestamp"] == 0.0
|
||||
|
||||
assert "FINAL_LONG_SUMMARY" in eventnames
|
||||
ev = events[eventnames.index("FINAL_LONG_SUMMARY")]
|
||||
assert ev["data"]["long_summary"] == "LLM LONG SUMMARY"
|
||||
|
||||
assert "FINAL_SHORT_SUMMARY" in eventnames
|
||||
ev = events[eventnames.index("FINAL_SHORT_SUMMARY")]
|
||||
assert ev["data"]["short_summary"] == "LLM SHORT SUMMARY"
|
||||
|
||||
assert "FINAL_TITLE" in eventnames
|
||||
ev = events[eventnames.index("FINAL_TITLE")]
|
||||
assert ev["data"]["title"] == "Llm Title"
|
||||
|
||||
# check status order
|
||||
# Live pipeline status progression
|
||||
statuses = [e["data"]["value"] for e in events if e["event"] == "STATUS"]
|
||||
assert "recording" in statuses
|
||||
assert "processing" in statuses
|
||||
assert statuses.index("recording") < statuses.index("processing")
|
||||
assert statuses.index("processing") < statuses.index("ended")
|
||||
|
||||
# ensure the last event received is ended
|
||||
assert events[-1]["event"] == "STATUS"
|
||||
assert events[-1]["data"]["value"] == "ended"
|
||||
# Post-processing (FINAL_*, STATUS=ended) is now dispatched to Hatchet
|
||||
# via LivePostPipeline — not tested here.
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import pytest
|
||||
|
||||
from reflector.settings import settings
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_create_default_translation(client):
|
||||
async def test_transcript_create_default_translation(monkeypatch, client):
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post("/transcripts", json={"name": "test en"})
|
||||
assert response.status_code == 200
|
||||
assert response.json()["name"] == "test en"
|
||||
@@ -18,7 +23,10 @@ async def test_transcript_create_default_translation(client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_create_en_fr_translation(client):
|
||||
async def test_transcript_create_en_fr_translation(monkeypatch, client):
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post(
|
||||
"/transcripts", json={"name": "test en/fr", "target_language": "fr"}
|
||||
)
|
||||
@@ -36,7 +44,10 @@ async def test_transcript_create_en_fr_translation(client):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_create_fr_en_translation(client):
|
||||
async def test_transcript_create_fr_en_translation(monkeypatch, client):
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
response = await client.post(
|
||||
"/transcripts", json={"name": "test fr/en", "source_language": "fr"}
|
||||
)
|
||||
|
||||
@@ -1,12 +1,7 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("setup_database")
|
||||
@pytest.mark.usefixtures("celery_session_app")
|
||||
@pytest.mark.usefixtures("celery_session_worker")
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_upload_file(
|
||||
tmpdir,
|
||||
@@ -16,7 +11,14 @@ async def test_transcript_upload_file(
|
||||
dummy_file_diarization,
|
||||
dummy_storage,
|
||||
client,
|
||||
monkeypatch,
|
||||
mock_hatchet_client,
|
||||
):
|
||||
from reflector.settings import settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "PUBLIC_MODE", True
|
||||
) # public mode: allow anonymous transcript creation for this test
|
||||
# create a transcript
|
||||
response = await client.post("/transcripts", json={"name": "test"})
|
||||
assert response.status_code == 200
|
||||
@@ -37,27 +39,16 @@ async def test_transcript_upload_file(
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "ok"
|
||||
|
||||
# wait the processing to finish (max 1 minute)
|
||||
timeout_seconds = 60
|
||||
start_time = time.monotonic()
|
||||
while (time.monotonic() - start_time) < timeout_seconds:
|
||||
# fetch the transcript and check if it is ended
|
||||
resp = await client.get(f"/transcripts/{tid}")
|
||||
assert resp.status_code == 200
|
||||
if resp.json()["status"] in ("ended", "error"):
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
else:
|
||||
return pytest.fail(f"Processing timed out after {timeout_seconds} seconds")
|
||||
# Verify Hatchet workflow was dispatched for file processing
|
||||
from reflector.hatchet.client import HatchetClientManager
|
||||
|
||||
# check the transcript is ended
|
||||
transcript = resp.json()
|
||||
assert transcript["status"] == "ended"
|
||||
assert transcript["short_summary"] == "LLM SHORT SUMMARY"
|
||||
assert transcript["title"] == "Llm Title"
|
||||
HatchetClientManager.start_workflow.assert_called_once_with(
|
||||
"FilePipeline",
|
||||
{"transcript_id": tid},
|
||||
additional_metadata={"transcript_id": tid},
|
||||
)
|
||||
|
||||
# check topics and transcript
|
||||
response = await client.get(f"/transcripts/{tid}/topics")
|
||||
assert response.status_code == 200
|
||||
assert len(response.json()) == 1
|
||||
assert "Hello world. How are you today?" in response.json()[0]["transcript"]
|
||||
# Verify transcript status was updated to "uploaded"
|
||||
resp = await client.get(f"/transcripts/{tid}")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "uploaded"
|
||||
|
||||
@@ -141,33 +141,19 @@ async def test_user_ws_accepts_valid_token_and_receives_events(appserver_ws_user
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
# Emit an event to the user's room via a standard HTTP action
|
||||
# Use a real HTTP request to the server with the JWT token so that
|
||||
# current_user_optional_if_public_mode is exercised without dependency overrides
|
||||
from httpx import AsyncClient
|
||||
|
||||
from reflector.app import app
|
||||
from reflector.auth import current_user, current_user_optional
|
||||
|
||||
# Override auth dependencies so HTTP request is performed as the same user
|
||||
# Use the internal user.id (not the Authentik UID)
|
||||
app.dependency_overrides[current_user] = lambda: {
|
||||
"sub": user.id,
|
||||
"email": "user-abc@example.com",
|
||||
}
|
||||
app.dependency_overrides[current_user_optional] = lambda: {
|
||||
"sub": user.id,
|
||||
"email": "user-abc@example.com",
|
||||
}
|
||||
|
||||
# Use in-memory client (global singleton makes it share ws_manager)
|
||||
async with AsyncClient(app=app, base_url=f"http://{host}:{port}/v1") as ac:
|
||||
# Create a transcript as this user so that the server publishes TRANSCRIPT_CREATED to user room
|
||||
resp = await ac.post("/transcripts", json={"name": "WS Test"})
|
||||
async with AsyncClient(base_url=f"http://{host}:{port}/v1") as ac:
|
||||
resp = await ac.post(
|
||||
"/transcripts",
|
||||
json={"name": "WS Test"},
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
# Receive the published event
|
||||
msg = await ws.receive_json()
|
||||
assert msg["event"] == "TRANSCRIPT_CREATED"
|
||||
assert "id" in msg["data"]
|
||||
|
||||
# Clean overrides
|
||||
del app.dependency_overrides[current_user]
|
||||
del app.dependency_overrides[current_user_optional]
|
||||
|
||||
84
server/uv.lock
generated
84
server/uv.lock
generated
@@ -404,7 +404,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "black"
|
||||
version = "24.3.0"
|
||||
version = "26.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
@@ -412,18 +412,21 @@ dependencies = [
|
||||
{ name = "packaging" },
|
||||
{ name = "pathspec" },
|
||||
{ name = "platformdirs" },
|
||||
{ name = "pytokens" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8f/5f/bac24a952668c7482cfdb4ebf91ba57a796c9da8829363a772040c1a3312/black-24.3.0.tar.gz", hash = "sha256:a0c9c4a0771afc6919578cec71ce82a3e31e054904e7197deacbc9382671c41f", size = 634292, upload-time = "2024-03-15T19:35:43.699Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e1/c5/61175d618685d42b005847464b8fb4743a67b1b8fdb75e50e5a96c31a27a/black-26.3.1.tar.gz", hash = "sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07", size = 666155, upload-time = "2026-03-12T03:36:03.593Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/68/df/ceea5828be9c4931cb5a75b7e8fb02971f57524da7a16dfec0d4d575327f/black-24.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4f1373a7808a8f135b774039f61d59e4be7eb56b2513d3d2f02a8b9365b8a8a9", size = 1571235, upload-time = "2024-03-15T19:45:27.77Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/5f/30398c5056cb72f883b32b6520ad00042a9d0454b693f70509867db03a80/black-24.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aadf7a02d947936ee418777e0247ea114f78aff0d0959461057cae8a04f20597", size = 1414926, upload-time = "2024-03-15T19:43:52.993Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/59/498885b279e890f656ea4300a2671c964acb6d97994ea626479c2e5501b4/black-24.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c02e4ea2ae09d16314d30912a58ada9a5c4fdfedf9512d23326128ac08ac3d", size = 1725920, upload-time = "2024-03-15T19:38:13.052Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8f/b0/4bef40c808cc615187db983b75bacdca1c110a229d41ba9887549fac529c/black-24.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:bf21b7b230718a5f08bd32d5e4f1db7fc8788345c8aea1d155fc17852b3410f5", size = 1372608, upload-time = "2024-03-15T19:39:34.973Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b6/c6/1d174efa9ff02b22d0124c73fc5f4d4fb006d0d9a081aadc354d05754a13/black-24.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2818cf72dfd5d289e48f37ccfa08b460bf469e67fb7c4abb07edc2e9f16fb63f", size = 1600822, upload-time = "2024-03-15T19:45:20.337Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/ed/704731afffe460b8ff0672623b40fce9fe569f2ee617c15857e4d4440a3a/black-24.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4acf672def7eb1725f41f38bf6bf425c8237248bb0804faa3965c036f7672d11", size = 1429987, upload-time = "2024-03-15T19:45:00.637Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/05/8dd038e30caadab7120176d4bc109b7ca2f4457f12eef746b0560a583458/black-24.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7ed6668cbbfcd231fa0dc1b137d3e40c04c7f786e626b405c62bcd5db5857e4", size = 1755319, upload-time = "2024-03-15T19:38:24.009Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/9d/e5fa1ff4ef1940be15a64883c0bb8d2fcf626efec996eab4ae5a8c691d2c/black-24.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:56f52cfbd3dabe2798d76dbdd299faa046a901041faf2cf33288bc4e6dae57b5", size = 1385180, upload-time = "2024-03-15T19:39:37.014Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4d/ea/31770a7e49f3eedfd8cd7b35e78b3a3aaad860400f8673994bc988318135/black-24.3.0-py3-none-any.whl", hash = "sha256:41622020d7120e01d377f74249e677039d20e6344ff5851de8a10f11f513bf93", size = 201493, upload-time = "2024-03-15T19:35:41.572Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/17/57/5f11c92861f9c92eb9dddf515530bc2d06db843e44bdcf1c83c1427824bc/black-26.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff", size = 1851987, upload-time = "2026-03-12T03:40:06.248Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/aa/340a1463660bf6831f9e39646bf774086dbd8ca7fc3cded9d59bbdf4ad0a/black-26.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c", size = 1689499, upload-time = "2026-03-12T03:40:07.642Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/01/b726c93d717d72733da031d2de10b92c9fa4c8d0c67e8a8a372076579279/black-26.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5", size = 1754369, upload-time = "2026-03-12T03:40:09.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/09/61e91881ca291f150cfc9eb7ba19473c2e59df28859a11a88248b5cbbc4d/black-26.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e", size = 1413613, upload-time = "2026-03-12T03:40:10.943Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/16/73/544f23891b22e7efe4d8f812371ab85b57f6a01b2fc45e3ba2e52ba985b8/black-26.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5", size = 1219719, upload-time = "2026-03-12T03:40:12.597Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/f8/da5eae4fc75e78e6dceb60624e1b9662ab00d6b452996046dfa9b8a6025b/black-26.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1", size = 1895920, upload-time = "2026-03-12T03:40:13.921Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/9f/04e6f26534da2e1629b2b48255c264cabf5eedc5141d04516d9d68a24111/black-26.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f", size = 1718499, upload-time = "2026-03-12T03:40:15.239Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/91/a5935b2a63e31b331060c4a9fdb5a6c725840858c599032a6f3aac94055f/black-26.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7", size = 1794994, upload-time = "2026-03-12T03:40:17.124Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/0a/86e462cdd311a3c2a8ece708d22aba17d0b2a0d5348ca34b40cdcbea512e/black-26.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983", size = 1420867, upload-time = "2026-03-12T03:40:18.83Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/e5/22515a19cb7eaee3440325a6b0d95d2c0e88dd180cb011b12ae488e031d1/black-26.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb", size = 1230124, upload-time = "2026-03-12T03:40:20.425Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/0d/52d98722666d6fc6c3dd4c76df339501d6efd40e0ff95e6186a7b7f0befd/black-26.3.1-py3-none-any.whl", hash = "sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b", size = 207542, upload-time = "2026-03-12T03:36:01.668Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2433,11 +2436,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pathspec"
|
||||
version = "0.12.1"
|
||||
version = "1.0.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2915,11 +2918,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pyjwt"
|
||||
version = "2.11.0"
|
||||
version = "2.12.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5c/5a/b46fa56bf322901eee5b0454a34343cdbdae202cd421775a8ee4e42fd519/pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623", size = 98019, upload-time = "2026-01-30T19:59:55.694Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c2/27/a3b6e5bf6ff856d2509292e95c8f57f0df7017cf5394921fc4e4ef40308a/pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b", size = 102564, upload-time = "2026-03-13T19:27:37.25Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/01/c26ce75ba460d5cd503da9e13b21a33804d38c2165dec7b716d06b13010c/pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469", size = 28224, upload-time = "2026-01-30T19:59:54.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/7a/8dd906bd22e79e47397a61742927f6747fe93242ef86645ee9092e610244/pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", size = 29726, upload-time = "2026-03-13T19:27:35.677Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
@@ -2951,15 +2954,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pyopenssl"
|
||||
version = "25.3.0"
|
||||
version = "26.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cryptography" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/80/be/97b83a464498a79103036bc74d1038df4a7ef0e402cfaf4d5e113fb14759/pyopenssl-25.3.0.tar.gz", hash = "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329", size = 184073, upload-time = "2025-09-17T00:32:21.037Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8e/11/a62e1d33b373da2b2c2cd9eb508147871c80f12b1cacde3c5d314922afdd/pyopenssl-26.0.0.tar.gz", hash = "sha256:f293934e52936f2e3413b89c6ce36df66a0b34ae1ea3a053b8c5020ff2f513fc", size = 185534, upload-time = "2026-03-15T14:28:26.353Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/81/ef2b1dfd1862567d573a4fdbc9f969067621764fbb74338496840a1d2977/pyopenssl-25.3.0-py3-none-any.whl", hash = "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6", size = 57268, upload-time = "2025-09-17T00:32:19.474Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/7d/d4f7d908fa8415571771b30669251d57c3cf313b36a856e6d7548ae01619/pyopenssl-26.0.0-py3-none-any.whl", hash = "sha256:df94d28498848b98cc1c0ffb8ef1e71e40210d3b0a8064c9d29571ed2904bf81", size = 57969, upload-time = "2026-03-15T14:28:24.864Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2973,11 +2976,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "pypdf"
|
||||
version = "6.7.5"
|
||||
version = "6.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f6/52/37cc0aa9e9d1bf7729a737a0d83f8b3f851c8eb137373d9f71eafb0a3405/pypdf-6.7.5.tar.gz", hash = "sha256:40bb2e2e872078655f12b9b89e2f900888bb505e88a82150b64f9f34fa25651d", size = 5304278, upload-time = "2026-03-02T09:05:21.464Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b4/a3/e705b0805212b663a4c27b861c8a603dba0f8b4bb281f96f8e746576a50d/pypdf-6.8.0.tar.gz", hash = "sha256:cb7eaeaa4133ce76f762184069a854e03f4d9a08568f0e0623f7ea810407833b", size = 5307831, upload-time = "2026-03-09T13:37:40.591Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/05/89/336673efd0a88956562658aba4f0bbef7cb92a6fbcbcaf94926dbc82b408/pypdf-6.7.5-py3-none-any.whl", hash = "sha256:07ba7f1d6e6d9aa2a17f5452e320a84718d4ce863367f7ede2fd72280349ab13", size = 331421, upload-time = "2026-03-02T09:05:19.722Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/ec/4ccf3bb86b1afe5d7176e1c8abcdbf22b53dd682ec2eda50e1caadcf6846/pypdf-6.8.0-py3-none-any.whl", hash = "sha256:2a025080a8dd73f48123c89c57174a5ff3806c71763ee4e49572dc90454943c7", size = 332177, upload-time = "2026-03-09T13:37:38.774Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3159,6 +3162,25 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579, upload-time = "2026-01-25T10:15:54.811Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytokens"
|
||||
version = "0.4.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/92/790ebe03f07b57e53b10884c329b9a1a308648fc083a6d4a39a10a28c8fc/pytokens-0.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d70e77c55ae8380c91c0c18dea05951482e263982911fc7410b1ffd1dadd3440", size = 160864, upload-time = "2026-01-30T01:02:57.882Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/25/a4f555281d975bfdd1eba731450e2fe3a95870274da73fb12c40aeae7625/pytokens-0.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a58d057208cb9075c144950d789511220b07636dd2e4708d5645d24de666bdc", size = 248565, upload-time = "2026-01-30T01:02:59.912Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/17/50/bc0394b4ad5b1601be22fa43652173d47e4c9efbf0044c62e9a59b747c56/pytokens-0.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b49750419d300e2b5a3813cf229d4e5a4c728dae470bcc89867a9ad6f25a722d", size = 260824, upload-time = "2026-01-30T01:03:01.471Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/54/3e04f9d92a4be4fc6c80016bc396b923d2a6933ae94b5f557c939c460ee0/pytokens-0.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d9907d61f15bf7261d7e775bd5d7ee4d2930e04424bab1972591918497623a16", size = 264075, upload-time = "2026-01-30T01:03:04.143Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/1b/44b0326cb5470a4375f37988aea5d61b5cc52407143303015ebee94abfd6/pytokens-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6", size = 103323, upload-time = "2026-01-30T01:03:05.412Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytorch-lightning"
|
||||
version = "2.5.6"
|
||||
@@ -4268,10 +4290,10 @@ dependencies = [
|
||||
{ name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torchaudio-2.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c9276857d241c6de257af765c0f51fc011af38cb725401495121b280913007cf" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torchaudio-2.8.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4573c6042950c20278e3608a9a38050ba0bc72e0049e1bbfd249caf859a8029b" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torchaudio-2.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ddef94bf181e6447cbb05f38beaca8f6c5bb8d2b9ddced1aa3452025b9fc70d3" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torchaudio-2.8.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:862e2e40bf09d865e5df080a84c1a39bbcef40e43140f4b1737eb3a389d3b38f" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchaudio-2.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c9276857d241c6de257af765c0f51fc011af38cb725401495121b280913007cf" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchaudio-2.8.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4573c6042950c20278e3608a9a38050ba0bc72e0049e1bbfd249caf859a8029b" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchaudio-2.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ddef94bf181e6447cbb05f38beaca8f6c5bb8d2b9ddced1aa3452025b9fc70d3" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchaudio-2.8.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:862e2e40bf09d865e5df080a84c1a39bbcef40e43140f4b1737eb3a389d3b38f" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4294,10 +4316,10 @@ dependencies = [
|
||||
{ name = "torch", version = "2.8.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation == 'PyPy' and sys_platform == 'darwin') or (platform_python_implementation != 'CPython' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torchaudio-2.8.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e54bd7fc9472019308097d99102df9acee22aa2451ae808d27840bc874320292" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torchaudio-2.8.0%2Bcpu-cp311-cp311-win_amd64.whl", hash = "sha256:db37df7eee906f8fe0a639fdc673f3541cb2e173169b16d4133447eb922d1938" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torchaudio-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9377faee65a290578280ac7f4884c3586253dac2ca28c60f458ff6efe86a6b05" },
|
||||
{ url = "https://download.pytorch.org/whl/cpu/torchaudio-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:9b302192b570657c1cc787a4d487ae4bbb7f2aab1c01b1fcc46757e7f86f391e" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchaudio-2.8.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e54bd7fc9472019308097d99102df9acee22aa2451ae808d27840bc874320292" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchaudio-2.8.0%2Bcpu-cp311-cp311-win_amd64.whl", hash = "sha256:db37df7eee906f8fe0a639fdc673f3541cb2e173169b16d4133447eb922d1938" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchaudio-2.8.0%2Bcpu-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9377faee65a290578280ac7f4884c3586253dac2ca28c60f458ff6efe86a6b05" },
|
||||
{ url = "https://download-r2.pytorch.org/whl/cpu/torchaudio-2.8.0%2Bcpu-cp312-cp312-win_amd64.whl", hash = "sha256:9b302192b570657c1cc787a4d487ae4bbb7f2aab1c01b1fcc46757e7f86f391e" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
minimum-release-age=1440 #24hr in minutes
|
||||
minimum-release-age=1440 #24hr in minutes
|
||||
minimum-release-age-exclude[]=next
|
||||
minimum-release-age-exclude[]=@next/*
|
||||
1
www/.prettierignore
Normal file
1
www/.prettierignore
Normal file
@@ -0,0 +1 @@
|
||||
pnpm-lock.yaml
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Reconnection policy for WebSocket.
|
||||
* Ensures exponential backoff is applied and capped at 30s.
|
||||
*/
|
||||
import { getReconnectDelayMs, MAX_RETRIES } from "../webSocketReconnect";
|
||||
|
||||
describe("webSocketReconnect", () => {
|
||||
describe("getReconnectDelayMs", () => {
|
||||
it("returns exponential backoff: 1s, 2s, 4s, 8s, 16s, then cap 30s", () => {
|
||||
expect(getReconnectDelayMs(0)).toBe(1000);
|
||||
expect(getReconnectDelayMs(1)).toBe(2000);
|
||||
expect(getReconnectDelayMs(2)).toBe(4000);
|
||||
expect(getReconnectDelayMs(3)).toBe(8000);
|
||||
expect(getReconnectDelayMs(4)).toBe(16000);
|
||||
expect(getReconnectDelayMs(5)).toBe(30000); // 32s capped to 30s
|
||||
expect(getReconnectDelayMs(6)).toBe(30000);
|
||||
expect(getReconnectDelayMs(9)).toBe(30000);
|
||||
});
|
||||
|
||||
it("never exceeds 30s for any retry index", () => {
|
||||
for (let i = 0; i <= MAX_RETRIES; i++) {
|
||||
expect(getReconnectDelayMs(i)).toBeLessThanOrEqual(30000);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
} from "../../lib/apiHooks";
|
||||
import { useAuth } from "../../lib/AuthProvider";
|
||||
import { parseNonEmptyString } from "../../lib/utils";
|
||||
import { getReconnectDelayMs, MAX_RETRIES } from "./webSocketReconnect";
|
||||
|
||||
type TranscriptWsEvent =
|
||||
operations["v1_transcript_get_websocket_events"]["responses"][200]["content"]["application/json"];
|
||||
@@ -338,7 +339,6 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
|
||||
if (!transcriptId) return;
|
||||
const tsId = parseNonEmptyString(transcriptId);
|
||||
|
||||
const MAX_RETRIES = 10;
|
||||
const url = `${WEBSOCKET_URL}/v1/transcripts/${transcriptId}/events`;
|
||||
let ws: WebSocket | null = null;
|
||||
let retryCount = 0;
|
||||
@@ -472,7 +472,7 @@ export const useWebSockets = (transcriptId: string | null): UseWebSockets => {
|
||||
if (normalCodes.includes(event.code)) return;
|
||||
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
const delay = Math.min(1000 * Math.pow(2, retryCount), 30000);
|
||||
const delay = getReconnectDelayMs(retryCount);
|
||||
console.log(
|
||||
`WebSocket reconnecting in ${delay}ms (attempt ${retryCount + 1}/${MAX_RETRIES})`,
|
||||
);
|
||||
|
||||
10
www/app/(app)/transcripts/webSocketReconnect.ts
Normal file
10
www/app/(app)/transcripts/webSocketReconnect.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
/** Reconnection policy for WebSocket: exponential backoff, capped at 30s. */
|
||||
export const MAX_RETRIES = 10;
|
||||
|
||||
/**
|
||||
* Delay in ms before reconnecting. retryIndex is 0-based (0 = first retry).
|
||||
* Returns 1000, 2000, 4000, ... up to 30000 max.
|
||||
*/
|
||||
export function getReconnectDelayMs(retryIndex: number): number {
|
||||
return Math.min(1000 * Math.pow(2, retryIndex), 30000);
|
||||
}
|
||||
@@ -10,7 +10,8 @@
|
||||
"lint": "next lint",
|
||||
"format": "prettier --write .",
|
||||
"openapi": "openapi-typescript http://127.0.0.1:1250/openapi.json -o ./app/reflector-api.d.ts",
|
||||
"test": "jest"
|
||||
"test": "jest",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@chakra-ui/react": "^3.33.0",
|
||||
@@ -30,7 +31,7 @@
|
||||
"ioredis": "^5.10.0",
|
||||
"jest-worker": "^30.2.0",
|
||||
"lucide-react": "^0.575.0",
|
||||
"next": "^16.1.6",
|
||||
"next": "16.1.7",
|
||||
"next-auth": "^4.24.13",
|
||||
"next-themes": "^0.4.6",
|
||||
"nuqs": "^2.8.9",
|
||||
@@ -74,7 +75,8 @@
|
||||
"js-yaml@<4.1.1": "4.1.1",
|
||||
"webpack": "5.105.3",
|
||||
"serialize-javascript": "7.0.4",
|
||||
"immutable": "5.1.5"
|
||||
"immutable": "5.1.5",
|
||||
"next": "16.1.7"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
12315
www/pnpm-lock.yaml
generated
12315
www/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user