From a22789d5486bf8b83e33ab2fb5eb3ee9799c6d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Diego=20Garc=C3=ADa?= Date: Mon, 30 Mar 2026 17:46:23 -0500 Subject: [PATCH] fix: grpc tls for local hatchet (#937) --- docsv2/custom-ca-setup.md | 3 ++- gpu/self_hosted/docker-entrypoint.sh | 3 ++- server/docker-entrypoint.sh | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docsv2/custom-ca-setup.md b/docsv2/custom-ca-setup.md index 142717bc..068f6c93 100644 --- a/docsv2/custom-ca-setup.md +++ b/docsv2/custom-ca-setup.md @@ -199,7 +199,8 @@ Each backend container (server, worker, beat, hatchet workers, GPU) has an entry | `SSL_CERT_FILE` | httpx, OpenAI SDK, llama-index, Python ssl module | | `REQUESTS_CA_BUNDLE` | requests library (transitive dependencies) | | `CURL_CA_BUNDLE` | curl CLI (container healthchecks) | -| `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH` | grpcio (Hatchet gRPC client) | + +Note: `GRPC_DEFAULT_SSL_ROOTS_FILE_PATH` is intentionally NOT set. Setting it causes grpcio to attempt TLS on internal Hatchet gRPC connections that run without TLS, resulting in handshake failures. The internal Hatchet connection uses `HATCHET_CLIENT_TLS_STRATEGY=none` (plaintext). When no CA cert is mounted, the entrypoint is a no-op — containers behave exactly as before. diff --git a/gpu/self_hosted/docker-entrypoint.sh b/gpu/self_hosted/docker-entrypoint.sh index aab14ad6..6f2f391b 100644 --- a/gpu/self_hosted/docker-entrypoint.sh +++ b/gpu/self_hosted/docker-entrypoint.sh @@ -15,7 +15,8 @@ if [ -s "$CUSTOM_CA_PATH" ]; then export SSL_CERT_FILE="$COMBINED_BUNDLE" export REQUESTS_CA_BUNDLE="$COMBINED_BUNDLE" export CURL_CA_BUNDLE="$COMBINED_BUNDLE" - export GRPC_DEFAULT_SSL_ROOTS_FILE_PATH="$COMBINED_BUNDLE" + # Note: GRPC_DEFAULT_SSL_ROOTS_FILE_PATH is intentionally NOT set here. + # Setting it causes grpcio to attempt TLS on connections that may be plaintext. echo "[entrypoint] CA trust store updated (SSL_CERT_FILE=$COMBINED_BUNDLE)" fi diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh index bfdaa1ad..d544dee9 100644 --- a/server/docker-entrypoint.sh +++ b/server/docker-entrypoint.sh @@ -15,7 +15,10 @@ if [ -s "$CUSTOM_CA_PATH" ]; then export SSL_CERT_FILE="$COMBINED_BUNDLE" export REQUESTS_CA_BUNDLE="$COMBINED_BUNDLE" export CURL_CA_BUNDLE="$COMBINED_BUNDLE" - export GRPC_DEFAULT_SSL_ROOTS_FILE_PATH="$COMBINED_BUNDLE" + # Note: GRPC_DEFAULT_SSL_ROOTS_FILE_PATH is intentionally NOT set here. + # Setting it causes grpcio to attempt TLS on internal Hatchet connections + # that run without TLS (SERVER_GRPC_INSECURE=t), resulting in handshake failures. + # If you need gRPC with custom CA, set GRPC_DEFAULT_SSL_ROOTS_FILE_PATH explicitly. echo "[entrypoint] CA trust store updated (SSL_CERT_FILE=$COMBINED_BUNDLE)" fi