fix(langfuse): fix goose langfuse integration (wrong env variables)

SPECIFICATIONS.md

@@ -229,8 +229,9 @@ logging:
     - type: fluentd
       url: http://fluentd.example.com:24224
     - type: langfuse
-      url: https://api.langfuse.com
+      url: https://cloud.langfuse.com
-      apiKey: ${LANGFUSE_API_KEY}
+      public_key: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY}
+      secret_key: ${LANGFUSE_INIT_PROJECT_SECRET_KEY}
 
 drivers:
   - name: goose
@@ -277,6 +278,43 @@ The MC Service implements log collection and forwarding:
 
 ## Project Management
 
+### Persistent Project Configuration
+
+MC provides persistent storage for project-specific configurations that need to survive container restarts. This is implemented through a dedicated volume mount and symlink system:
+
+1. **Configuration Storage**:
+   - Each project has a dedicated configuration directory on the host at `~/.mc/projects/<project-hash>/config`
+   - For projects specified by URL, the hash is derived from the repository URL
+   - For local projects, the hash is derived from the absolute path of the local directory
+   - This directory is mounted into the container at `/mc-config`
+
+2. **Driver Configuration**:
+   - Each driver can specify configuration files/directories that should persist across sessions
+   - These are defined in the driver's `mc-driver.yaml` file in the `persistent_configs` section
+   - Example for Goose driver:
+     ```yaml
+     persistent_configs:
+       - source: "/app/.goose"         # Path in container
+         target: "/mc-config/goose"    # Path in persistent storage
+         type: "directory"             # directory or file
+         description: "Goose memory and configuration"
+     ```
+
+3. **Automatic Symlinking**:
+   - During container initialization, the system:
+     - Creates all target directories in the persistent storage
+     - Creates symlinks from the source paths to the target paths
+   - This makes the persistence transparent to the application
+
+4. **Environment Variables**:
+   - Container has access to configuration location via environment variables:
+     ```
+     MC_CONFIG_DIR=/mc-config
+     MC_DRIVER_CONFIG_DIR=/mc-config/<driver-name>
+     ```
+
+This ensures that important configurations like Goose's memory store, authentication tokens, and other state information persist between container sessions while maintaining isolation between different projects.
+
 ### Adding Projects
 
 Users can add projects with associated credentials:
@@ -415,6 +453,12 @@ ports:
 volumes:
   - mountPath: /app
     description: Application directory
+
+persistent_configs:
+  - source: "/app/.goose"
+    target: "/mc-config/goose"
+    type: "directory"
+    description: "Goose memory and configuration"
 ```
 
 ### Example Built-in Drivers

drivers/goose/README.md

@@ -15,11 +15,9 @@ This driver provides a containerized environment for running [Goose](https://goo
 | Variable | Description | Required |
 |----------|-------------|----------|
 | `MCP_HOST` | MCP server host | Yes |
-| `GOOSE_API_KEY` | Goose API key | Yes |
+| `LANGFUSE_INIT_PROJECT_PUBLIC_KEY` | Langfuse public key | No |
-| `GOOSE_ID` | Goose instance ID | No |
+| `LANGFUSE_INIT_PROJECT_SECRET_KEY` | Langfuse secret key | No |
-| `LANGFUSE_PUBLIC_KEY` | Langfuse public key | No |
+| `LANGFUSE_URL` | Langfuse API URL | No |
-| `LANGFUSE_SECRET_KEY` | Langfuse secret key | No |
-| `LANGFUSE_HOST` | Langfuse API host | No |
 | `MC_PROJECT_URL` | Project repository URL | No |
 | `MC_GIT_SSH_KEY` | SSH key for Git authentication | No |
 | `MC_GIT_TOKEN` | Token for Git authentication | No |

drivers/goose/mc-driver.yaml

@@ -13,29 +13,20 @@ environment:
     required: true
     default: http://localhost:8000
 
-  - name: GOOSE_API_KEY
+  - name: LANGFUSE_INIT_PROJECT_PUBLIC_KEY
-    description: Goose API key
-    required: true
-    sensitive: true
-
-  - name: GOOSE_ID
-    description: Goose instance ID
-    required: false
-
-  - name: LANGFUSE_PUBLIC_KEY
     description: Langfuse public key
     required: false
     sensitive: true
 
-  - name: LANGFUSE_SECRET_KEY
+  - name: LANGFUSE_INIT_PROJECT_SECRET_KEY
     description: Langfuse secret key
     required: false
     sensitive: true
 
-  - name: LANGFUSE_HOST
+  - name: LANGFUSE_URL
-    description: Langfuse API host
+    description: Langfuse API URL
     required: false
-    default: https://api.langfuse.com
+    default: https://cloud.langfuse.com
 
   # Project environment variables
   - name: MC_PROJECT_URL
@@ -63,4 +54,10 @@ ports:
 
 volumes:
   - mountPath: /app
-    description: Application directory
+    description: Application directory
+
+persistent_configs:
+  - source: "/app/.goose"
+    target: "/mc-config/goose"
+    type: "directory"
+    description: "Goose memory and configuration"

drivers/goose/mc-init.sh

@@ -8,6 +8,26 @@ exec > >(tee -a /init.log) 2>&1
 echo "=== MC Initialization started at $(date) ==="
 echo "INIT_COMPLETE=false" > /init.status
 
+# Set up persistent configuration symlinks
+if [ -n "$MC_CONFIG_DIR" ] && [ -d "$MC_CONFIG_DIR" ]; then
+    echo "Setting up persistent configuration in $MC_CONFIG_DIR"
+
+    # Create Goose configuration directory
+    mkdir -p "$MC_CONFIG_DIR/goose"
+
+    # Create symlink for Goose directory
+    if [ -d "/app" ]; then
+        # Make sure .goose directory exists in the target
+        mkdir -p "$MC_CONFIG_DIR/goose"
+
+        # Create the symlink
+        echo "Creating symlink for Goose configuration: /app/.goose -> $MC_CONFIG_DIR/goose"
+        ln -sf "$MC_CONFIG_DIR/goose" "/app/.goose"
+    else
+        echo "Warning: /app directory does not exist yet, symlinks will be created after project initialization"
+    fi
+fi
+
 # Project initialization
 if [ -n "$MC_PROJECT_URL" ]; then
     echo "Initializing project: $MC_PROJECT_URL"
@@ -36,13 +56,21 @@ if [ -n "$MC_PROJECT_URL" ]; then
     if [ -f "/app/.mc/init.sh" ]; then
         bash /app/.mc/init.sh
     fi
+
+    # Set up symlinks after project is cloned (if MC_CONFIG_DIR exists)
+    if [ -n "$MC_CONFIG_DIR" ] && [ -d "$MC_CONFIG_DIR" ]; then
+        echo "Setting up persistent configuration symlinks after project clone"
+
+        # Create Goose configuration directory
+        mkdir -p "$MC_CONFIG_DIR/goose"
+
+        # Create symlink for Goose directory
+        echo "Creating symlink for Goose configuration: /app/.goose -> $MC_CONFIG_DIR/goose"
+        ln -sf "$MC_CONFIG_DIR/goose" "/app/.goose"
+    fi
 fi
 
-# Set up Goose API key if provided
+# Goose uses self-hosted instance, no API key required
-if [ -n "$GOOSE_API_KEY" ]; then
-    echo "Setting up Goose API key"
-    export GOOSE_API_KEY="$GOOSE_API_KEY"
-fi
 
 # Set up MCP connection if provided
 if [ -n "$MCP_HOST" ]; then
@@ -51,11 +79,11 @@ if [ -n "$MCP_HOST" ]; then
 fi
 
 # Set up Langfuse logging if credentials are provided
-if [ -n "$LANGFUSE_SECRET_KEY" ] && [ -n "$LANGFUSE_PUBLIC_KEY" ]; then
+if [ -n "$LANGFUSE_INIT_PROJECT_SECRET_KEY" ] && [ -n "$LANGFUSE_INIT_PROJECT_PUBLIC_KEY" ]; then
     echo "Setting up Langfuse logging"
-    export LANGFUSE_SECRET_KEY="$LANGFUSE_SECRET_KEY"
+    export LANGFUSE_INIT_PROJECT_SECRET_KEY="$LANGFUSE_INIT_PROJECT_SECRET_KEY"
-    export LANGFUSE_PUBLIC_KEY="$LANGFUSE_PUBLIC_KEY"
+    export LANGFUSE_INIT_PROJECT_PUBLIC_KEY="$LANGFUSE_INIT_PROJECT_PUBLIC_KEY"
-    export LANGFUSE_HOST="${LANGFUSE_HOST:-https://api.langfuse.com}"
+    export LANGFUSE_URL="${LANGFUSE_URL:-https://cloud.langfuse.com}"
 fi
 
 echo "MC driver initialization complete"

mcontainer/container.py

@@ -2,6 +2,8 @@ import os
 import sys
 import uuid
 import docker
+import hashlib
+import pathlib
 import concurrent.futures
 from typing import Dict, List, Optional, Tuple
 from docker.errors import DockerException, ImageNotFound
@@ -32,6 +34,38 @@ class ContainerManager:
         """Generate a unique session ID"""
         return str(uuid.uuid4())[:8]
 
+    def _get_project_config_path(self, project: Optional[str] = None) -> pathlib.Path:
+        """Get the path to the project configuration directory
+
+        Args:
+            project: Optional project repository URL. If None, uses current directory.
+
+        Returns:
+            Path to the project configuration directory
+        """
+        # Get home directory for the MC config
+        mc_home = pathlib.Path.home() / ".mc"
+
+        # If no project URL is provided, use the current directory path
+        if not project:
+            # Use current working directory as project identifier
+            project_id = os.getcwd()
+        else:
+            # Use project URL as identifier
+            project_id = project
+
+        # Create a hash of the project ID to use as directory name
+        project_hash = hashlib.md5(project_id.encode()).hexdigest()
+
+        # Create the project config directory path
+        config_path = mc_home / "projects" / project_hash / "config"
+
+        # Create the directory if it doesn't exist
+        config_path.parent.mkdir(parents=True, exist_ok=True)
+        config_path.mkdir(exist_ok=True)
+
+        return config_path
+
     def list_sessions(self) -> List[Session]:
         """List all active MC sessions"""
         sessions = []
@@ -124,7 +158,14 @@ class ContainerManager:
                 env_vars["MC_PROJECT_URL"] = project
 
             # Pass API keys from host environment to container for local development
-            api_keys = ["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "OPENROUTER_API_KEY"]
+            api_keys = [
+                "OPENAI_API_KEY",
+                "ANTHROPIC_API_KEY",
+                "OPENROUTER_API_KEY",
+                "LANGFUSE_INIT_PROJECT_PUBLIC_KEY",
+                "LANGFUSE_INIT_PROJECT_SECRET_KEY",
+                "LANGFUSE_URL",
+            ]
             for key in api_keys:
                 if key in os.environ and key not in env_vars:
                     env_vars[key] = os.environ[key]
@@ -150,6 +191,36 @@ class ContainerManager:
                     f"Project URL provided - container will clone {project} into /app during initialization"
                 )
 
+            # Set up persistent project configuration
+            project_config_path = self._get_project_config_path(project)
+            print(f"Using project configuration directory: {project_config_path}")
+
+            # Mount the project configuration directory
+            volumes[str(project_config_path)] = {"bind": "/mc-config", "mode": "rw"}
+
+            # Add environment variables for config path
+            env_vars["MC_CONFIG_DIR"] = "/mc-config"
+            env_vars["MC_DRIVER_CONFIG_DIR"] = f"/mc-config/{driver_name}"
+
+            # Create driver-specific config directories
+            if driver.persistent_configs:
+                print("Setting up persistent configuration directories:")
+                for config in driver.persistent_configs:
+                    # Get target directory path on host
+                    target_dir = project_config_path / config.target.lstrip(
+                        "/mc-config/"
+                    )
+
+                    # Create directory if it's a directory type config
+                    if config.type == "directory":
+                        target_dir.mkdir(parents=True, exist_ok=True)
+                        print(f"  - Created directory: {target_dir}")
+
+                    # For files, make sure parent directory exists
+                    elif config.type == "file":
+                        target_dir.parent.mkdir(parents=True, exist_ok=True)
+                        # File will be created by the container if needed
+
             # Create container
             container = self.client.containers.create(
                 image=driver.image,

mcontainer/models.py

@@ -18,6 +18,13 @@ class DriverEnvironmentVariable(BaseModel):
     sensitive: bool = False
 
 
+class PersistentConfig(BaseModel):
+    source: str
+    target: str
+    type: str  # "directory" or "file"
+    description: str = ""
+
+
 class Driver(BaseModel):
     name: str
     description: str
@@ -27,6 +34,7 @@ class Driver(BaseModel):
     environment: List[DriverEnvironmentVariable] = []
     ports: List[int] = []
     volumes: List[Dict[str, str]] = []
+    persistent_configs: List[PersistentConfig] = []
 
 
 class Session(BaseModel):