From f4f94a0d9998030e5ef7f01935d99722045165ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Diego=20Garc=C3=ADa?= Date: Tue, 7 Apr 2026 17:14:55 -0500 Subject: [PATCH] feat: allow livekit to work with tunnels udp and tcp with quickguide (#948) --- docker-compose.selfhosted.yml | 2 +- docsv2/tunnel-setup.md | 144 ++++++++++++++++++ scripts/setup-selfhosted.sh | 122 ++++++++++++++- .../integration/test_multitrack_pipeline.py | 2 +- 4 files changed, 261 insertions(+), 9 deletions(-) create mode 100644 docsv2/tunnel-setup.md diff --git a/docker-compose.selfhosted.yml b/docker-compose.selfhosted.yml index ef606459..dda5185a 100644 --- a/docker-compose.selfhosted.yml +++ b/docker-compose.selfhosted.yml @@ -418,7 +418,7 @@ services: ports: - "7880:7880" # Signaling (HTTP/WS) - "7881:7881" # WebRTC over TCP - - "44200-44300:44200-44300/udp" # WebRTC ICE (avoids macOS ephemeral 49152-65535 and Reflector 40000-40100) + - "${LIVEKIT_UDP_PORTS:-44200-44300:44200-44300}/udp" # WebRTC ICE (range or single port for tunnels) volumes: - ./livekit.yaml:/etc/livekit.yaml:ro command: ["--config", "/etc/livekit.yaml"] diff --git a/docsv2/tunnel-setup.md b/docsv2/tunnel-setup.md new file mode 100644 index 00000000..63739f05 --- /dev/null +++ b/docsv2/tunnel-setup.md @@ -0,0 +1,144 @@ +# Tunnel Setup (Self-Hosting Behind NAT) + +Expose your self-hosted Reflector + LiveKit stack to the internet without port forwarding, static IPs, or cloud VMs using tunneling services. + +## Requirements + +You need **two tunnels**: + +| Tunnel | Protocol | What it carries | Local port | Examples | +|--------|----------|----------------|------------|----------| +| **TCP tunnel** | TCP | Web app, API, LiveKit signaling (WebSocket) | 443 (Caddy) | playit.gg, ngrok, Cloudflare Tunnel, bore, frp | +| **UDP tunnel** | UDP | WebRTC audio/video media | Assigned by tunnel service | playit.gg, frp | + +> **Important:** Most tunneling services only support TCP. WebRTC media requires UDP. Make sure your chosen service supports UDP tunnels. As of writing, [playit.gg](https://playit.gg) is one of the few that supports both TCP and UDP (premium $3/mo). + +## Architecture + +``` +Internet participants + │ + ├── TCP tunnel (HTTPS) + │ └── tunnel service → your machine port 443 (Caddy) + │ ├── /v1/* → server:1250 (API) + │ ├── /lk-ws/* → livekit-server:7880 (signaling) + │ └── /* → web:3000 (frontend) + │ + └── UDP tunnel + └── tunnel service → your machine port N (LiveKit ICE) +``` + +## Setup + +### Step 1: Create tunnels with your chosen service + +Create two tunnels and note the public addresses: + +- **TCP tunnel**: Points to your local port `443` + - You'll get an address like `your-tunnel.example.com:PORT` +- **UDP tunnel**: Points to a local port (e.g., `14139`) + - You'll get an address like `udp-host.example.com:PORT` + - **The local port must match the public port** (or LiveKit ICE candidates won't match). Set the local port to the same number as the public port assigned by the tunnel service. + +### Step 2: Run the setup script + +```bash +./scripts/setup-selfhosted.sh --livekit --garage \ + --tunnels , +``` + +Example: +```bash +./scripts/setup-selfhosted.sh --cpu --livekit --garage \ + --tunnels my-tunnel.example.com:9055,udp-host.example.com:14139 +``` + +Or use separate flags: +```bash +./scripts/setup-selfhosted.sh --cpu --livekit --garage \ + --tunnel-tcp my-tunnel.example.com:9055 \ + --tunnel-udp udp-host.example.com:14139 +``` + +The script automatically: +- Sets all URLs (API, frontend, LiveKit signaling) to the TCP tunnel address +- Configures LiveKit with the UDP tunnel port and resolved IP for ICE candidates +- Enables Caddy with self-signed TLS (catch-all on port 443) +- Saves tunnel config for re-runs + +### Step 3: Start the tunnel agent + +Run your tunneling service's agent/client on the same machine. It must be running whenever you want external access. + +### Step 4: Access + +Share `https://` with participants. They'll need to accept the self-signed certificate warning in their browser. + +## Flag Reference + +| Flag | Description | +|------|-------------| +| `--tunnels TCP,UDP` | Both tunnel addresses comma-separated (e.g., `host:9055,host:14139`) | +| `--tunnel-tcp ADDR` | TCP tunnel address only (e.g., `host.example.com:9055`) | +| `--tunnel-udp ADDR` | UDP tunnel address only (e.g., `host.example.com:14139`) | + +Tunnel flags: +- Imply `--caddy` (HTTPS required for browser mic/camera access) +- Are mutually exclusive with `--ip` and `--domain` +- Are saved to config memory (re-run without flags replays saved config) + +## UDP Port Matching + +LiveKit advertises ICE candidates with a specific IP and port. The browser connects to that exact address. If the tunnel's public port differs from the local port, ICE will fail. + +**Correct setup:** Set the tunnel's local port to match its public port. + +``` +Tunnel assigns public port 14139 + → Set local port to 14139 + → LiveKit listens on 14139 (udp_port in livekit.yaml) + → Docker maps 14139:14139/udp + → ICE candidates advertise tunnel_ip:14139 + → Browser connects to tunnel_ip:14139 → tunnel → local:14139 → LiveKit +``` + +If your tunneling service doesn't let you choose the local port, you'll need to update `livekit.yaml` manually with the assigned ports. + +## TLS Certificate Warning + +With tunnel services on non-standard ports (e.g., `:9055`), Let's Encrypt can't auto-provision certificates (it requires ports 80/443). Caddy uses `tls internal` which generates a self-signed certificate. Participants will see a browser warning they must accept. + +**To avoid the warning:** +- Use a tunnel service that provides port 443 for TCP +- Or use a real domain with `--domain` on a server with a public IP + +## Compatible Tunnel Services + +| Service | TCP | UDP | Free tier | Notes | +|---------|-----|-----|-----------|-------| +| [playit.gg](https://playit.gg) | Yes (premium) | Yes (premium) | Limited | $3/mo premium. Supports both TCP + UDP. | +| [ngrok](https://ngrok.com) | Yes | No | Limited | TCP only — needs a separate UDP tunnel for media | +| [Cloudflare Tunnel](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/) | Yes | No | Yes | TCP only — needs a separate UDP tunnel for media | +| [bore](https://github.com/ekzhang/bore) | Yes | No | Self-hosted | TCP only | +| [frp](https://github.com/fatedier/frp) | Yes | Yes | Self-hosted | Requires your own VPS to run the frp server | +| [Tailscale Funnel](https://tailscale.com/kb/1223/funnel) | Yes | No | Free (3 nodes) | TCP only, requires Tailscale account | + +For a full self-contained setup without a VPS, playit.gg (TCP + UDP) is currently the simplest option. + +## Limitations + +- **Latency**: Adds a hop through the tunnel service's relay servers +- **Bandwidth**: Tunnel services may have bandwidth limits on free/cheap tiers +- **Reliability**: Depends on the tunnel service's uptime +- **Certificate warning**: Unavoidable with non-standard ports (see above) +- **Single UDP port**: Tunnel mode uses a single UDP port instead of a range, which limits concurrent WebRTC connections (~50 participants max) +- **Not production-grade**: Suitable for demos, small teams, development, and privacy-first setups. For production, use a server with a public IP. + +## Comparison + +| Approach | Cost | Setup | Data location | Port forwarding needed | +|----------|------|-------|---------------|----------------------| +| **Tunnel (this guide)** | $0-3/mo | Low | Your machine | No | +| **Cloud VM** | $5-20/mo | Low | Cloud provider | No | +| **Port forwarding** | $0 | Medium | Your machine | Yes (router config) | +| **VPN mesh (Tailscale)** | $0 | Low | Your machine | No (VPN peers only) | diff --git a/scripts/setup-selfhosted.sh b/scripts/setup-selfhosted.sh index f4e2011f..8676ee29 100755 --- a/scripts/setup-selfhosted.sh +++ b/scripts/setup-selfhosted.sh @@ -32,6 +32,11 @@ # (self-signed HTTPS, required for browser mic/camera access). # Mutually exclusive with --domain. Use for LAN or cloud VM access. # On Linux, IP is auto-detected; on macOS, use --ip to specify it. +# --tunnels TCP,UDP Configure tunnel addresses for NAT traversal (e.g. playit.gg). +# TCP=host:port for web/API/signaling, UDP=host:port for WebRTC media. +# Implies --caddy. Mutually exclusive with --ip and --domain. +# --tunnel-tcp ADDR TCP tunnel address only (e.g. --tunnel-tcp host.playit.gg:9055) +# --tunnel-udp ADDR UDP tunnel address only (e.g. --tunnel-udp host.ply.gg:14139) # --garage Use Garage for local S3-compatible storage # --caddy Enable Caddy reverse proxy with auto-SSL # --domain DOMAIN Use a real domain for Caddy (enables Let's Encrypt auto-HTTPS) @@ -220,6 +225,8 @@ USE_CADDY=false CUSTOM_DOMAIN="" # optional domain for Let's Encrypt HTTPS CUSTOM_IP="" # optional --ip override (mutually exclusive with --caddy) BUILD_IMAGES=false # build backend/frontend from source +TUNNEL_TCP="" # --tunnel-tcp: TCP tunnel address (e.g., host:port from playit.gg) +TUNNEL_UDP="" # --tunnel-udp: UDP tunnel address (e.g., host:port from playit.gg) ADMIN_PASSWORD="" # optional admin password for password auth CUSTOM_CA="" # --custom-ca: path to dir or CA cert file USE_CUSTOM_CA=false # derived flag: true when --custom-ca is provided @@ -282,6 +289,33 @@ for i in "${!ARGS[@]}"; do CUSTOM_IP="${ARGS[$next_i]}" SKIP_NEXT=true ;; --build) BUILD_IMAGES=true ;; + --tunnels) + next_i=$((i + 1)) + if [[ $next_i -ge ${#ARGS[@]} ]] || [[ "${ARGS[$next_i]}" == --* ]]; then + err "--tunnels requires TCP,UDP addresses (e.g. --tunnels host:9055,host:14139)" + exit 1 + fi + IFS=',' read -r TUNNEL_TCP TUNNEL_UDP <<< "${ARGS[$next_i]}" + # Trim whitespace + TUNNEL_TCP="${TUNNEL_TCP// /}" + TUNNEL_UDP="${TUNNEL_UDP// /}" + SKIP_NEXT=true ;; + --tunnel-tcp) + next_i=$((i + 1)) + if [[ $next_i -ge ${#ARGS[@]} ]] || [[ "${ARGS[$next_i]}" == --* ]]; then + err "--tunnel-tcp requires a TCP tunnel address (e.g. --tunnel-tcp host:9055)" + exit 1 + fi + TUNNEL_TCP="${ARGS[$next_i]}" + SKIP_NEXT=true ;; + --tunnel-udp) + next_i=$((i + 1)) + if [[ $next_i -ge ${#ARGS[@]} ]] || [[ "${ARGS[$next_i]}" == --* ]]; then + err "--tunnel-udp requires a UDP tunnel address (e.g. --tunnel-udp host:14139)" + exit 1 + fi + TUNNEL_UDP="${ARGS[$next_i]}" + SKIP_NEXT=true ;; --password) next_i=$((i + 1)) if [[ $next_i -ge ${#ARGS[@]} ]] || [[ "${ARGS[$next_i]}" == --* ]]; then @@ -384,6 +418,35 @@ fi if [[ -n "$CUSTOM_IP" ]]; then USE_CADDY=true fi +# Validate tunnel address format (must be host:port with numeric port) +_validate_tunnel_addr() { + local label="$1" addr="$2" + if [[ -z "$addr" ]]; then return; fi + if [[ "$addr" != *:* ]]; then + err "$label address must be host:port (got: $addr)" + exit 1 + fi + local port="${addr##*:}" + if ! [[ "$port" =~ ^[0-9]+$ ]] || [[ "$port" -lt 1 ]] || [[ "$port" -gt 65535 ]]; then + err "$label port must be 1-65535 (got: $port)" + exit 1 + fi +} +_validate_tunnel_addr "--tunnel-tcp" "$TUNNEL_TCP" +_validate_tunnel_addr "--tunnel-udp" "$TUNNEL_UDP" + +# --tunnels / --tunnel-tcp implies --caddy +if [[ -n "$TUNNEL_TCP" ]]; then + USE_CADDY=true +fi +if [[ -n "$TUNNEL_TCP" ]] && [[ -n "$CUSTOM_DOMAIN" ]]; then + err "--tunnel-tcp and --domain are mutually exclusive." + exit 1 +fi +if [[ -n "$TUNNEL_TCP" ]] && [[ -n "$CUSTOM_IP" ]]; then + err "--tunnel-tcp and --ip are mutually exclusive." + exit 1 +fi # --- Save CLI args for config memory (re-run without flags) --- if [[ $# -gt 0 ]]; then @@ -563,7 +626,10 @@ _generate_livekit_config() { # at /lk-ws to avoid mixed-content blocking (browsers block ws:// on https:// pages). # When no Caddy, browsers connect directly to LiveKit on port 7880. local public_lk_url - if [[ "$USE_CADDY" == "true" ]]; then + if [[ -n "$TUNNEL_TCP" ]]; then + # Tunnel mode: LiveKit signaling proxied through Caddy on the tunnel address + public_lk_url="wss://${TUNNEL_TCP}/lk-ws" + elif [[ "$USE_CADDY" == "true" ]]; then if [[ -n "$CUSTOM_DOMAIN" ]]; then public_lk_url="wss://${CUSTOM_DOMAIN}/lk-ws" elif [[ -n "$PRIMARY_IP" ]]; then @@ -603,12 +669,35 @@ _generate_livekit_config() { fi # Generate livekit.yaml + # UDP tunnel mode: use single port matching the tunnel's public port + local rtc_config + if [[ -n "$TUNNEL_UDP" ]]; then + local tunnel_udp_host tunnel_udp_port + tunnel_udp_host="${TUNNEL_UDP%:*}" + tunnel_udp_port="${TUNNEL_UDP##*:}" + # Resolve tunnel hostname to IP for node_ip + local tunnel_udp_ip + tunnel_udp_ip=$(dig +short "$tunnel_udp_host" 2>/dev/null | head -1 || nslookup "$tunnel_udp_host" 2>/dev/null | grep "Address:" | tail -1 | awk '{print $2}' || true) + if [[ -z "$tunnel_udp_ip" ]]; then + warn "Could not resolve UDP tunnel hostname: $tunnel_udp_host" + warn "Set node_ip manually in livekit.yaml after setup" + tunnel_udp_ip="0.0.0.0" + fi + rtc_config=" tcp_port: 7881 + udp_port: ${tunnel_udp_port} + node_ip: ${tunnel_udp_ip} + use_external_ip: false" + ok "LiveKit UDP: single port ${tunnel_udp_port}, node_ip=${tunnel_udp_ip} (via tunnel)" + else + rtc_config=" tcp_port: 7881 + port_range_start: 44200 + port_range_end: 44300" + fi + cat > "$ROOT_DIR/livekit.yaml" << LKEOF port: 7880 rtc: - tcp_port: 7881 - port_range_start: 44200 - port_range_end: 44300 +${rtc_config} redis: address: redis:6379 keys: @@ -862,7 +951,10 @@ step_server_env() { # Public-facing URLs local server_base_url - if [[ -n "$CUSTOM_DOMAIN" ]]; then + if [[ -n "$TUNNEL_TCP" ]]; then + # Tunnel mode: public URL is the tunnel address with HTTPS (Caddy terminates TLS) + server_base_url="https://$TUNNEL_TCP" + elif [[ -n "$CUSTOM_DOMAIN" ]]; then server_base_url="https://$CUSTOM_DOMAIN" elif [[ "$USE_CADDY" == "true" ]]; then if [[ -n "$PRIMARY_IP" ]]; then @@ -1111,6 +1203,16 @@ step_server_env() { ok "BIND_HOST=0.0.0.0 (ports exposed for direct access)" fi + # UDP ports for LiveKit (used by docker-compose for port mapping) + if [[ -n "$TUNNEL_UDP" ]]; then + local tunnel_udp_port="${TUNNEL_UDP##*:}" + env_set "$root_env" "LIVEKIT_UDP_PORTS" "${tunnel_udp_port}:${tunnel_udp_port}" + ok "LiveKit UDP: single port ${tunnel_udp_port} (via tunnel)" + else + # Default: full range for direct access + env_set "$root_env" "LIVEKIT_UDP_PORTS" "44200-44300:44200-44300" + fi + ok "server/.env ready" } @@ -1129,7 +1231,9 @@ step_www_env() { # Public-facing URL for frontend local base_url - if [[ -n "$CUSTOM_DOMAIN" ]]; then + if [[ -n "$TUNNEL_TCP" ]]; then + base_url="https://$TUNNEL_TCP" + elif [[ -n "$CUSTOM_DOMAIN" ]]; then base_url="https://$CUSTOM_DOMAIN" elif [[ "$USE_CADDY" == "true" ]]; then if [[ -n "$PRIMARY_IP" ]]; then @@ -1905,7 +2009,11 @@ EOF [[ "$USES_OLLAMA" != "true" ]] && echo " LLM: External (configure in server/.env)" [[ "$DAILY_DETECTED" == "true" ]] && echo " Video: Daily.co (live rooms + multitrack processing via Hatchet)" [[ "$WHEREBY_DETECTED" == "true" ]] && echo " Video: Whereby (live rooms)" - [[ "$LIVEKIT_DETECTED" == "true" ]] && echo " Video: LiveKit (self-hosted, live rooms + track egress)" + if [[ "$LIVEKIT_DETECTED" == "true" ]]; then + echo " Video: LiveKit (self-hosted, live rooms + track egress)" + [[ -n "$TUNNEL_TCP" ]] && echo " Tunnel: TCP=$TUNNEL_TCP" + [[ -n "$TUNNEL_UDP" ]] && echo " UDP=$TUNNEL_UDP" + fi [[ "$ANY_PLATFORM_DETECTED" != "true" ]] && echo " Video: None (rooms disabled)" if [[ "$USE_CUSTOM_CA" == "true" ]]; then echo " CA: Custom (certs/ca.crt)" diff --git a/server/tests/integration/test_multitrack_pipeline.py b/server/tests/integration/test_multitrack_pipeline.py index 1a8a9257..e3561ba3 100644 --- a/server/tests/integration/test_multitrack_pipeline.py +++ b/server/tests/integration/test_multitrack_pipeline.py @@ -171,5 +171,5 @@ async def test_multitrack_pipeline_end_to_end( assert len(messages) >= 1, "Should have received at least 1 email" email_msg = messages[0] assert ( - "Transcript Ready" in email_msg.get("Subject", "") + "Reflector:" in email_msg.get("Subject", "") ), f"Email subject should contain 'Transcript Ready', got: {email_msg.get('Subject')}"