diff --git a/SPECIFICATIONS.md b/SPECIFICATIONS.md index febabda..f85431c 100644 --- a/SPECIFICATIONS.md +++ b/SPECIFICATIONS.md @@ -229,8 +229,9 @@ logging: - type: fluentd url: http://fluentd.example.com:24224 - type: langfuse - url: https://api.langfuse.com - apiKey: ${LANGFUSE_API_KEY} + url: https://cloud.langfuse.com + public_key: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY} + secret_key: ${LANGFUSE_INIT_PROJECT_SECRET_KEY} drivers: - name: goose @@ -277,6 +278,43 @@ The MC Service implements log collection and forwarding: ## Project Management +### Persistent Project Configuration + +MC provides persistent storage for project-specific configurations that need to survive container restarts. This is implemented through a dedicated volume mount and symlink system: + +1. **Configuration Storage**: + - Each project has a dedicated configuration directory on the host at `~/.mc/projects//config` + - For projects specified by URL, the hash is derived from the repository URL + - For local projects, the hash is derived from the absolute path of the local directory + - This directory is mounted into the container at `/mc-config` + +2. **Driver Configuration**: + - Each driver can specify configuration files/directories that should persist across sessions + - These are defined in the driver's `mc-driver.yaml` file in the `persistent_configs` section + - Example for Goose driver: + ```yaml + persistent_configs: + - source: "/app/.goose" # Path in container + target: "/mc-config/goose" # Path in persistent storage + type: "directory" # directory or file + description: "Goose memory and configuration" + ``` + +3. **Automatic Symlinking**: + - During container initialization, the system: + - Creates all target directories in the persistent storage + - Creates symlinks from the source paths to the target paths + - This makes the persistence transparent to the application + +4. **Environment Variables**: + - Container has access to configuration location via environment variables: + ``` + MC_CONFIG_DIR=/mc-config + MC_DRIVER_CONFIG_DIR=/mc-config/ + ``` + +This ensures that important configurations like Goose's memory store, authentication tokens, and other state information persist between container sessions while maintaining isolation between different projects. + ### Adding Projects Users can add projects with associated credentials: @@ -415,6 +453,12 @@ ports: volumes: - mountPath: /app description: Application directory + +persistent_configs: + - source: "/app/.goose" + target: "/mc-config/goose" + type: "directory" + description: "Goose memory and configuration" ``` ### Example Built-in Drivers diff --git a/drivers/goose/README.md b/drivers/goose/README.md index 7f68e79..204f9a4 100644 --- a/drivers/goose/README.md +++ b/drivers/goose/README.md @@ -15,11 +15,9 @@ This driver provides a containerized environment for running [Goose](https://goo | Variable | Description | Required | |----------|-------------|----------| | `MCP_HOST` | MCP server host | Yes | -| `GOOSE_API_KEY` | Goose API key | Yes | -| `GOOSE_ID` | Goose instance ID | No | -| `LANGFUSE_PUBLIC_KEY` | Langfuse public key | No | -| `LANGFUSE_SECRET_KEY` | Langfuse secret key | No | -| `LANGFUSE_HOST` | Langfuse API host | No | +| `LANGFUSE_INIT_PROJECT_PUBLIC_KEY` | Langfuse public key | No | +| `LANGFUSE_INIT_PROJECT_SECRET_KEY` | Langfuse secret key | No | +| `LANGFUSE_URL` | Langfuse API URL | No | | `MC_PROJECT_URL` | Project repository URL | No | | `MC_GIT_SSH_KEY` | SSH key for Git authentication | No | | `MC_GIT_TOKEN` | Token for Git authentication | No | diff --git a/drivers/goose/mc-driver.yaml b/drivers/goose/mc-driver.yaml index ca62461..99ba1dd 100644 --- a/drivers/goose/mc-driver.yaml +++ b/drivers/goose/mc-driver.yaml @@ -13,29 +13,20 @@ environment: required: true default: http://localhost:8000 - - name: GOOSE_API_KEY - description: Goose API key - required: true - sensitive: true - - - name: GOOSE_ID - description: Goose instance ID - required: false - - - name: LANGFUSE_PUBLIC_KEY + - name: LANGFUSE_INIT_PROJECT_PUBLIC_KEY description: Langfuse public key required: false sensitive: true - - name: LANGFUSE_SECRET_KEY + - name: LANGFUSE_INIT_PROJECT_SECRET_KEY description: Langfuse secret key required: false sensitive: true - - name: LANGFUSE_HOST - description: Langfuse API host + - name: LANGFUSE_URL + description: Langfuse API URL required: false - default: https://api.langfuse.com + default: https://cloud.langfuse.com # Project environment variables - name: MC_PROJECT_URL @@ -63,4 +54,10 @@ ports: volumes: - mountPath: /app - description: Application directory \ No newline at end of file + description: Application directory + +persistent_configs: + - source: "/app/.goose" + target: "/mc-config/goose" + type: "directory" + description: "Goose memory and configuration" \ No newline at end of file diff --git a/drivers/goose/mc-init.sh b/drivers/goose/mc-init.sh index d40bfc9..9c7fa6f 100755 --- a/drivers/goose/mc-init.sh +++ b/drivers/goose/mc-init.sh @@ -8,6 +8,26 @@ exec > >(tee -a /init.log) 2>&1 echo "=== MC Initialization started at $(date) ===" echo "INIT_COMPLETE=false" > /init.status +# Set up persistent configuration symlinks +if [ -n "$MC_CONFIG_DIR" ] && [ -d "$MC_CONFIG_DIR" ]; then + echo "Setting up persistent configuration in $MC_CONFIG_DIR" + + # Create Goose configuration directory + mkdir -p "$MC_CONFIG_DIR/goose" + + # Create symlink for Goose directory + if [ -d "/app" ]; then + # Make sure .goose directory exists in the target + mkdir -p "$MC_CONFIG_DIR/goose" + + # Create the symlink + echo "Creating symlink for Goose configuration: /app/.goose -> $MC_CONFIG_DIR/goose" + ln -sf "$MC_CONFIG_DIR/goose" "/app/.goose" + else + echo "Warning: /app directory does not exist yet, symlinks will be created after project initialization" + fi +fi + # Project initialization if [ -n "$MC_PROJECT_URL" ]; then echo "Initializing project: $MC_PROJECT_URL" @@ -36,13 +56,21 @@ if [ -n "$MC_PROJECT_URL" ]; then if [ -f "/app/.mc/init.sh" ]; then bash /app/.mc/init.sh fi + + # Set up symlinks after project is cloned (if MC_CONFIG_DIR exists) + if [ -n "$MC_CONFIG_DIR" ] && [ -d "$MC_CONFIG_DIR" ]; then + echo "Setting up persistent configuration symlinks after project clone" + + # Create Goose configuration directory + mkdir -p "$MC_CONFIG_DIR/goose" + + # Create symlink for Goose directory + echo "Creating symlink for Goose configuration: /app/.goose -> $MC_CONFIG_DIR/goose" + ln -sf "$MC_CONFIG_DIR/goose" "/app/.goose" + fi fi -# Set up Goose API key if provided -if [ -n "$GOOSE_API_KEY" ]; then - echo "Setting up Goose API key" - export GOOSE_API_KEY="$GOOSE_API_KEY" -fi +# Goose uses self-hosted instance, no API key required # Set up MCP connection if provided if [ -n "$MCP_HOST" ]; then @@ -51,11 +79,11 @@ if [ -n "$MCP_HOST" ]; then fi # Set up Langfuse logging if credentials are provided -if [ -n "$LANGFUSE_SECRET_KEY" ] && [ -n "$LANGFUSE_PUBLIC_KEY" ]; then +if [ -n "$LANGFUSE_INIT_PROJECT_SECRET_KEY" ] && [ -n "$LANGFUSE_INIT_PROJECT_PUBLIC_KEY" ]; then echo "Setting up Langfuse logging" - export LANGFUSE_SECRET_KEY="$LANGFUSE_SECRET_KEY" - export LANGFUSE_PUBLIC_KEY="$LANGFUSE_PUBLIC_KEY" - export LANGFUSE_HOST="${LANGFUSE_HOST:-https://api.langfuse.com}" + export LANGFUSE_INIT_PROJECT_SECRET_KEY="$LANGFUSE_INIT_PROJECT_SECRET_KEY" + export LANGFUSE_INIT_PROJECT_PUBLIC_KEY="$LANGFUSE_INIT_PROJECT_PUBLIC_KEY" + export LANGFUSE_URL="${LANGFUSE_URL:-https://cloud.langfuse.com}" fi echo "MC driver initialization complete" diff --git a/mcontainer/container.py b/mcontainer/container.py index a728b19..574461e 100644 --- a/mcontainer/container.py +++ b/mcontainer/container.py @@ -2,6 +2,8 @@ import os import sys import uuid import docker +import hashlib +import pathlib import concurrent.futures from typing import Dict, List, Optional, Tuple from docker.errors import DockerException, ImageNotFound @@ -32,6 +34,38 @@ class ContainerManager: """Generate a unique session ID""" return str(uuid.uuid4())[:8] + def _get_project_config_path(self, project: Optional[str] = None) -> pathlib.Path: + """Get the path to the project configuration directory + + Args: + project: Optional project repository URL. If None, uses current directory. + + Returns: + Path to the project configuration directory + """ + # Get home directory for the MC config + mc_home = pathlib.Path.home() / ".mc" + + # If no project URL is provided, use the current directory path + if not project: + # Use current working directory as project identifier + project_id = os.getcwd() + else: + # Use project URL as identifier + project_id = project + + # Create a hash of the project ID to use as directory name + project_hash = hashlib.md5(project_id.encode()).hexdigest() + + # Create the project config directory path + config_path = mc_home / "projects" / project_hash / "config" + + # Create the directory if it doesn't exist + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.mkdir(exist_ok=True) + + return config_path + def list_sessions(self) -> List[Session]: """List all active MC sessions""" sessions = [] @@ -124,7 +158,14 @@ class ContainerManager: env_vars["MC_PROJECT_URL"] = project # Pass API keys from host environment to container for local development - api_keys = ["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "OPENROUTER_API_KEY"] + api_keys = [ + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + "LANGFUSE_INIT_PROJECT_PUBLIC_KEY", + "LANGFUSE_INIT_PROJECT_SECRET_KEY", + "LANGFUSE_URL", + ] for key in api_keys: if key in os.environ and key not in env_vars: env_vars[key] = os.environ[key] @@ -150,6 +191,36 @@ class ContainerManager: f"Project URL provided - container will clone {project} into /app during initialization" ) + # Set up persistent project configuration + project_config_path = self._get_project_config_path(project) + print(f"Using project configuration directory: {project_config_path}") + + # Mount the project configuration directory + volumes[str(project_config_path)] = {"bind": "/mc-config", "mode": "rw"} + + # Add environment variables for config path + env_vars["MC_CONFIG_DIR"] = "/mc-config" + env_vars["MC_DRIVER_CONFIG_DIR"] = f"/mc-config/{driver_name}" + + # Create driver-specific config directories + if driver.persistent_configs: + print("Setting up persistent configuration directories:") + for config in driver.persistent_configs: + # Get target directory path on host + target_dir = project_config_path / config.target.lstrip( + "/mc-config/" + ) + + # Create directory if it's a directory type config + if config.type == "directory": + target_dir.mkdir(parents=True, exist_ok=True) + print(f" - Created directory: {target_dir}") + + # For files, make sure parent directory exists + elif config.type == "file": + target_dir.parent.mkdir(parents=True, exist_ok=True) + # File will be created by the container if needed + # Create container container = self.client.containers.create( image=driver.image, diff --git a/mcontainer/models.py b/mcontainer/models.py index 51bbfd2..fe02360 100644 --- a/mcontainer/models.py +++ b/mcontainer/models.py @@ -18,6 +18,13 @@ class DriverEnvironmentVariable(BaseModel): sensitive: bool = False +class PersistentConfig(BaseModel): + source: str + target: str + type: str # "directory" or "file" + description: str = "" + + class Driver(BaseModel): name: str description: str @@ -27,6 +34,7 @@ class Driver(BaseModel): environment: List[DriverEnvironmentVariable] = [] ports: List[int] = [] volumes: List[Dict[str, str]] = [] + persistent_configs: List[PersistentConfig] = [] class Session(BaseModel):