From 3b2afe0deb1c50e1e8544ccb7f1ff5d98aa61eb7 Mon Sep 17 00:00:00 2001 From: James Brunton Date: Tue, 21 Apr 2026 16:18:25 +0100 Subject: [PATCH] Change `engine/.env` to be committed and have `.env.local` override (#6150) # Description of Changes We keep adding stuff to `engine/config/.env.example` and have to manually update `.env` because of it, which is really clunky, especially when working on multiple worktrees at once. This PR changes it so that we just have a committed `.env` file and have an `.env.local` override to put the actual private keys into, which should make it a bit easier to manage. > [!warning] > > After this goes in, be very careful for a little while not to accidentally commit any keys that you've got inside your `.env` file! --- .dockerignore | 1 + .gitignore | 1 + .taskfiles/engine.yml | 3 +- engine/{config/.env.example => .env} | 6 ++++ engine/.gitignore | 1 - engine/Dockerfile | 3 +- engine/scripts/setup_env.py | 50 +++++++------------------- engine/src/stirling/config/settings.py | 4 ++- engine/src/stirling/rag/README.md | 9 ++++- 9 files changed, 34 insertions(+), 44 deletions(-) rename engine/{config/.env.example => .env} (81%) diff --git a/.dockerignore b/.dockerignore index 72943ddcc0..ee5c23bed8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -65,6 +65,7 @@ README* .env .env.* !.env.example +!engine/.env # Misc *.swp diff --git a/.gitignore b/.gitignore index 7a632d1a8b..d6dc4826ca 100644 --- a/.gitignore +++ b/.gitignore @@ -165,6 +165,7 @@ __pycache__/ # Virtual environments .env* !.env*.example +!engine/.env .venv* env*/ venv*/ diff --git a/.taskfiles/engine.yml b/.taskfiles/engine.yml index cb357a9dc5..9d830e0808 100644 --- a/.taskfiles/engine.yml +++ b/.taskfiles/engine.yml @@ -20,9 +20,8 @@ tasks: - uv run scripts/setup_env.py sources: - scripts/setup_env.py - - config/.env.example generates: - - .env + - .env.local run: desc: "Run engine server" diff --git a/engine/config/.env.example b/engine/.env similarity index 81% rename from engine/config/.env.example rename to engine/.env index 5431066640..7d1aa13940 100644 --- a/engine/config/.env.example +++ b/engine/.env @@ -1,3 +1,9 @@ +############################################################################### +# Environment variables used within the AI Engine. +# Values can be overridden in the uncommitted sibling `.env.local` file. +# Note: This file is committed to Git, so should not contain any private keys. +############################################################################### + # Configure the model strings passed to pydantic-ai. Provider credentials are handled by # pydantic-ai and should be set using the provider's native environment variables, for example # ANTHROPIC_API_KEY or OPENAI_API_KEY. diff --git a/engine/.gitignore b/engine/.gitignore index 1116f48815..890e5247ba 100644 --- a/engine/.gitignore +++ b/engine/.gitignore @@ -19,7 +19,6 @@ yarn-error.log* .vite/ # Environment -.env .env.local # LaTeX outputs diff --git a/engine/Dockerfile b/engine/Dockerfile index 191d579fa9..5caeb5d6bf 100644 --- a/engine/Dockerfile +++ b/engine/Dockerfile @@ -12,9 +12,8 @@ RUN apt-get update \ WORKDIR /app -COPY pyproject.toml uv.lock Taskfile.yml ./ +COPY pyproject.toml uv.lock Taskfile.yml .env ./ COPY .taskfiles/ ./.taskfiles/ -COPY config/ ./config/ COPY scripts/ ./scripts/ RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-dev diff --git a/engine/scripts/setup_env.py b/engine/scripts/setup_env.py index 9626e2c8cf..c459679238 100644 --- a/engine/scripts/setup_env.py +++ b/engine/scripts/setup_env.py @@ -1,48 +1,24 @@ """ -Copies .env from .env.example if missing, and errors if any keys from the example -are absent from the actual .env file. +Ensures `.env.local` exists so developers have a place to put overrides +(API keys, local model choices, etc.) without touching the committed `.env`. Usage: uv run scripts/setup_env.py """ -import os -import shutil -import sys from pathlib import Path -from dotenv import dotenv_values - ROOT = Path(__file__).parent.parent -EXAMPLE_FILE = ROOT / "config" / ".env.example" -ENV_FILE = ROOT / ".env" +ENV_LOCAL_FILE = ROOT / ".env.local" -print("setup-env: see engine/config/.env.example for documentation") +TEMPLATE = """\ +############################################################################### +# Local overrides for `engine/.env` +# Put API keys and machine-specific settings here. Any variable defined here +# takes precedence over the committed `.env` +############################################################################### +""" -if not EXAMPLE_FILE.exists(): - print(f"setup-env: {EXAMPLE_FILE.name} not found, skipping", file=sys.stderr) - sys.exit(0) - -if not ENV_FILE.exists(): - shutil.copy(EXAMPLE_FILE, ENV_FILE) - print("setup-env: created .env from .env.example") - -env_keys = set(dotenv_values(ENV_FILE).keys()) | set(os.environ.keys()) -example_keys = set(dotenv_values(EXAMPLE_FILE).keys()) -missing = sorted(example_keys - env_keys) - -if missing: - sys.exit( - "setup-env: .env is missing keys from .env.example:\n" - + "\n".join(f" {k}" for k in missing) - + "\n Add them manually or delete your local .env to re-copy from config/.env.example." - ) - -extra = sorted(k for k in dotenv_values(ENV_FILE) if k.startswith("STIRLING_") and k not in example_keys) -if extra: - print( - "setup-env: .env contains STIRLING_ keys not in config/.env.example:\n" - + "\n".join(f" {k}" for k in extra) - + "\n Add them to config/.env.example if they are intentional.", - file=sys.stderr, - ) +if not ENV_LOCAL_FILE.exists(): + ENV_LOCAL_FILE.write_text(TEMPLATE) + print("setup-env: created empty .env.local for local overrides") diff --git a/engine/src/stirling/config/settings.py b/engine/src/stirling/config/settings.py index 2452b8ec8c..d4e6212283 100644 --- a/engine/src/stirling/config/settings.py +++ b/engine/src/stirling/config/settings.py @@ -12,6 +12,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict ENGINE_ROOT = Path(__file__).resolve().parents[3] ENV_FILE = ENGINE_ROOT / ".env" +ENV_LOCAL_FILE = ENGINE_ROOT / ".env.local" class RagBackend(StrEnum): @@ -20,7 +21,7 @@ class RagBackend(StrEnum): class AppSettings(BaseSettings): - model_config = SettingsConfigDict(env_file=ENV_FILE, extra="ignore", populate_by_name=True) + model_config = SettingsConfigDict(env_file=(ENV_FILE, ENV_LOCAL_FILE), extra="ignore", populate_by_name=True) smart_model_name: str = Field(validation_alias="STIRLING_SMART_MODEL") fast_model_name: str = Field(validation_alias="STIRLING_FAST_MODEL") @@ -74,6 +75,7 @@ def _configure_logging(level_name: str, log_file: str) -> None: @lru_cache(maxsize=1) def load_settings() -> AppSettings: load_dotenv(ENV_FILE) + load_dotenv(ENV_LOCAL_FILE, override=True) settings = AppSettings.model_validate({}) _configure_logging(settings.log_level, settings.log_file) return settings diff --git a/engine/src/stirling/rag/README.md b/engine/src/stirling/rag/README.md index a4bdb6c452..309a3a7c9b 100644 --- a/engine/src/stirling/rag/README.md +++ b/engine/src/stirling/rag/README.md @@ -37,7 +37,9 @@ multi = RagCapability(runtime.rag_service, collections=["company-docs", "product everything = RagCapability(runtime.rag_service) ``` -## Config (.env) +## Config + +Non-secret defaults live in the committed `engine/.env`: ``` STIRLING_RAG_BACKEND=sqlite # or "pgvector" @@ -47,6 +49,11 @@ STIRLING_RAG_PGVECTOR_DSN= # used when backend=pgvector STIRLING_RAG_CHUNK_SIZE=512 STIRLING_RAG_CHUNK_OVERLAP=64 STIRLING_RAG_TOP_K=5 +``` + +Provider credentials (and any local overrides) go in the uncommitted `engine/.env.local`: + +``` VOYAGE_API_KEY=your-key ```