Initial docs setup

.devcontainer/Dockerfile

@@ -2,10 +2,15 @@ FROM python:3.13-bookworm
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
-    USER=compiler
+    USER=compiler \
+    USER_UID=1000 \
+    USER_GID=1000
 
 # Install system dependencies and patch OS packages to reduce known vulnerabilities
-RUN apt-get update && apt-get upgrade -y --no-install-recommends && apt-get install -y --no-install-recommends \
+# using the archives and lists cache
+RUN --mount=type=cache,id=apt-archives,sharing=locked,target=/var/cache/apt/archives \
+    --mount=type=cache,id=apt-lists,sharing=locked,target=/var/lib/apt/lists \
+    apt-get update && apt-get upgrade -y --no-install-recommends && apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
     git \
@@ -43,8 +48,20 @@ USER $USER
 WORKDIR /home/$USER/src
 
 # Update PATH for local pip installs
-ENV PATH="$PATH:/home/$USER/.local/bin"
+ENV PATH="$PATH:/home/$USER/.local/bin" \
+    # update env for local pip installs
+    # see https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE
+    # since all `pip install` commands are in the context of $USER
+    # $PYTHONUSERBASE is the location used by default
+    PYTHONUSERBASE="/home/${USER}/.local" \
+    # where to store the pip cache (use the default)
+    # https://pip.pypa.io/en/stable/cli/pip/#cmdoption-cache-dir
+    PIP_CACHE_DIR="/home/${USER}/.cache/pip"
 
-# Update pip and install pre-commit
-RUN python -m pip install --upgrade pip && \
-    python -m pip install --no-cache-dir pre-commit
+COPY docs/requirements.txt docs/requirements.txt
+
+# Update pip and install Python tooling
+RUN --mount=type=cache,id=pipcache,target=${PIP_CACHE_DIR},uid=${USER_UID},gid=${USER_GID} \
+    python -m pip install --user --upgrade pip && \
+    python -m pip install --user pre-commit && \
+    python -m pip install --user -r docs/requirements.txt

.devcontainer/devcontainer.json

@@ -2,7 +2,7 @@
   "name": "compilerla/dashboards",
   "dockerComposeFile": ["../compose.yml"],
   "service": "dev",
-  "runServices": ["dev"],
+  "runServices": ["dev", "docs"],
   "workspaceFolder": "/home/compiler/src",
   "postCreateCommand": ["/bin/bash", ".devcontainer/postCreate.sh"],
   "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"],

.github/workflows/mkdocs.yml

@@ -0,0 +1,99 @@
+name: Publish docs
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/mkdocs.yml"
+      - "docs/**"
+      - "mkdocs.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/mkdocs.yml"
+      - "docs/**"
+      - "mkdocs.yml"
+
+jobs:
+  docs-preview:
+    name: Publish docs preview
+    runs-on: ubuntu-latest
+    if: github.event.pull_request
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: "refs/pull/${{ github.event.number }}/merge"
+
+      - name: Setup Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.13"
+          cache: pip
+          cache-dependency-path: "docs/requirements.txt"
+
+      - name: Build MkDocs website
+        run: |
+          pip install -r docs/requirements.txt
+          mkdocs build
+
+      - name: Install Netlify CLI
+        run: npm install --location=global netlify-cli@17.x.x
+
+      - name: Deploy Preview to Netlify
+        run: |
+          netlify deploy \
+            --alias="${GITHUB_REPOSITORY#*/}-${{ github.event.number }}" \
+            --auth=${{ secrets.NETLIFY_AUTH_TOKEN }} \
+            --dir="site" \
+            --site=${{ vars.NETLIFY_PREVIEW_APP_SITE_ID }}
+
+      - name: Find existing comment
+        uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0
+        id: find-comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: "Preview url: https://"
+
+      - name: Add Netlify link PR comment
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        if: steps.find-comment.outputs.comment-id == ''
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const hostnameSuffix = "compiler-previews.netlify.app"
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `Preview url: https://${context.repo.repo}-${{ github.event.number }}--${hostnameSuffix}`,
+            })
+
+  docs:
+    name: Publish docs
+    runs-on: ubuntu-latest
+    if: github.event.pull_request == null && github.event.push.branch == 'main'
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - uses: actions/setup-python@46b300bd3776ad20ef3e26060101221095800131 # v6.0.0
+        with:
+          python-version: "3.13"
+          cache: pip
+          cache-dependency-path: "docs/requirements.txt"
+
+      - name: Install MkDocs requirements
+        run: |
+          pip install -r docs/requirements.txt
+
+      - name: Deploy MkDocs website
+        run: |
+          mkdocs gh-deploy --force

AGENTS.md

@@ -11,6 +11,8 @@
 - **Terraform Execution**: ALWAYS run terraform commands via `exec` against the existing, running `dev` compose service. Do not run them directly on the host and do not run them in a new ephemeral `dev` service. This is where GCP credentials will be configured.
 - **Verification**: Run `terraform fmt` and `terraform validate` after every terraform file addition, modification, etc.
 - **Pre-commit**: Hooks are installed automatically via `.devcontainer/postCreate.sh`. Run `pre-commit run --all --all-files` after making file modifications.
+- **GitHub Actions**: ALWAYS use the latest full commit SHA reference for external steps, with a comment of the current version number.
+- **Docs**: When making changes, the docs should be checked and updated accordingly if necessary.
 
 ## Constraints & Gotchas
 

README.md

@@ -2,81 +2,10 @@
 
 Compiler's analytics and business intelligence workspace, powered by open-source [Metabase](https://www.metabase.com/).
 
----
+## Documentation
 
-## Local Development (VS Code Dev Containers)
+For detailed instructions on local development, architecture, and repository structure, please visit our documentation site:
 
-This repository follows Compiler's cross-platform [Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers) standard to ensure consistent development environments across macOS, Linux, and Windows.
+[docs.compiler.la/dashboards](https://docs.compiler.la/dashboards)
 
-### Prerequisites
-
-1. [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine running locally.
-2. [Visual Studio Code](https://code.visualstudio.com/).
-3. VS Code [Dev Containers Extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers).
-
-### Getting Started
-
-1. Copy the sample environment file:
-
-   ```bash
-   cp .env.example .env
-   ```
-
-2. Open this folder in VS Code.
-3. When prompted, click **"Reopen in Container"** (or press `F1` / `Ctrl+Shift+P` / `Cmd+Shift+P` and run `Dev Containers: Reopen in Container`).
-4. VS Code will build the dev container and launch all three services:
-   - **`dev`**: Your development shell containing `terraform`, `gcloud`, `pre-commit`, and Postgres client tools.
-   - **`postgres`**: Local PostgreSQL 16 database storing Metabase application state.
-   - **`metabase`**: Official Metabase application available at [http://localhost:3000](http://localhost:3000).
-
-### Helper Scripts (`bin/`)
-
-The following helper scripts are available to run inside the dev container:
-
-- **Wait for Metabase to initialize**:
-
-  ```bash
-  ./bin/wait-for-metabase.sh
-  ```
-
-- **Connect to the local Postgres metadata database**:
-
-  ```bash
-  ./bin/psql.sh
-  ```
-
----
-
-## Architecture Overview
-
-Production is deployed on **Google Cloud Platform (GCP)**:
-
-- **Cloud Run**: Runs the Metabase container image (`metabase/metabase:vX.Y.Z`) with minimum 1 instance to support background tasks and scheduled pulses.
-- **Cloud SQL (PostgreSQL)**: Stores Metabase application metadata (dashboards, questions, users, permissions).
-- **BigQuery**: Analytics warehouse in project `data-warehouse-492522`, queried via native IAM Service Account credentials.
-- **Cloudflare**: DNS management and SSL termination for `dashboards.compiler.la`.
-
-For the complete technical specification, see [plan.md](plan.md).
-
----
-
-## Repository Structure
-
-```text
-.
-├── .devcontainer/
-│   ├── devcontainer.json            # VS Code dev container definition
-│   ├── Dockerfile                   # Dev container image (Terraform, gcloud, pre-commit)
-│   └── postCreate.sh                # Container initialization & hook setup
-├── .github/
-│   └── workflows/                   # GitHub Actions for Terraform automation
-├── bin/                             # Helper bash scripts for inside the dev container
-│   ├── psql.sh                      # Connect to local PostgreSQL
-│   └── wait-for-metabase.sh         # Health check waiter
-├── compose.yml                      # Dev container service + local Metabase & Postgres
-├── .env.example                     # Sample local environment variables
-├── .pre-commit-config.yaml          # Conventional commits & file hygiene hooks
-├── plan.md                          # Full architectural design & implementation plan
-├── terraform/                       # Infrastructure as Code (GCP Cloud Run & Cloud SQL)
-└── README.md
-```
+_Note: The documentation site is available at <http://localhost:8001> locally when running the `docs` service via `compose.yml`._

compose.yml

@@ -20,7 +20,14 @@ services:
         condition: "service_healthy"
       metabase:
         condition: "service_started"
-
+  docs:
+    image: "compilerla/dashboards:dev"
+    entrypoint: mkdocs
+    command: serve --livereload --dev-addr "0.0.0.0:8001"
+    ports:
+      - "8001:8001"
+    volumes:
+      - "./:/home/compiler/src"
   metabase:
     image: "metabase/metabase:v0.65.x"
     container_name: "metabase"
@@ -39,7 +46,6 @@ services:
       postgres:
         condition: "service_healthy"
     restart: "unless-stopped"
-
   postgres:
     image: "postgres:18-alpine"
     container_name: "postgres"
@@ -58,3 +64,4 @@ services:
 
 volumes:
   postgres_data:
+    driver: local

docs/explanation/architecture.md

@@ -0,0 +1,65 @@
+# Project Architecture
+
+This section explains the reasoning behind our design choices.
+
+## Architecture Overview
+
+```mermaid
+flowchart TD
+    subgraph Users & Ingress
+        U_Internal["Internal Users (Google Workspace SSO)"]
+        U_External["External Users (Username / Password)"]
+        CF["Cloudflare DNS & Proxy (dashboards.compiler.la)"]
+    end
+
+    subgraph GCP ["Google Cloud Platform (Project: data-warehouse-492522 | Region: us-west1)"]
+        subgraph CloudRun ["Cloud Run (Managed Container Runtime)"]
+            MB["Metabase OSS Container<br/>(metabase/metabase:vX.Y.Z)<br/>Min Instances: 1 | 2 GiB RAM"]
+        end
+
+        subgraph Storage_State ["Application State & Secrets"]
+            CSQL[("Cloud SQL PostgreSQL<br/>(Application Metadata & Dashboards)")]
+            SM["Secret Manager<br/>(DB Pass, Encryption Key, OAuth Secret)"]
+        end
+
+        subgraph Analytics ["Analytics Warehouse"]
+            BQ[("Google BigQuery (data-warehouse-492522)")]
+        end
+
+        subgraph IAM_Service ["Security & Identity"]
+            SA["Cloud Run Service Account<br/>(roles/bigquery.jobUser<br/>roles/bigquery.dataViewer)"]
+        end
+    end
+
+    subgraph Outbound ["Notifications"]
+        GW_SMTP["Google Workspace SMTP Relay<br/>(Invitations, Alerts, Password Resets)"]
+    end
+
+    U_Internal --> CF
+    U_External --> CF
+    CF -->|HTTPS| MB
+
+    MB -->|Cloud SQL Connection Unix Socket| CSQL
+    MB -.->|Fetch Secrets on Boot| SM
+    MB -->|Native IAM Auth / Zero Keys| BQ
+    SA -.->|Attached Identity| MB
+
+    MB -->|Port 587 TLS| GW_SMTP
+```
+
+## Key Architecture Decisions
+
+| Component                | Selected Configuration                 | Rationale                                                                                                                                      |
+| :----------------------- | :------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------- |
+| **GCP Project**          | `data-warehouse-492522`                | Host project containing BigQuery analytics warehouse and infrastructure.                                                                       |
+| **GCP Region**           | `us-west1` (Oregon)                    | Optimal low-latency to California; standard Tier 1 GCP pricing (matches `us-central1` costs).                                                  |
+| **Domain & DNS**         | `dashboards.compiler.la` (Cloudflare)  | Free managed SSL certificates; CNAME pointing to Cloud Run; proxy/WAF protection.                                                              |
+| **Container Runtime**    | **GCP Cloud Run**                      | Zero Kubernetes overhead; fully managed container lifecycle; integrated Cloud SQL proxy; native HTTPS.                                         |
+| **Instance Policy**      | **Min Instances: 1**                   | Prevents cold-start delays and ensures Metabase background syncs, alerts, and scheduled pulses run continuously.                               |
+| **Application Database** | **Cloud SQL PostgreSQL**               | Production-ready state store for Metabase (dashboards, queries, users, permissions). Never use the default embedded H2 database in production. |
+| **Analytics Warehouse**  | **Google BigQuery**                    | Native driver built into Metabase. Authenticated via the Cloud Run attached Service Account (no JSON key rotation needed).                     |
+| **Authentication**       | **Dual Auth: Google OAuth + Password** | Google Workspace OAuth restricted to `@compiler.la` for internal staff, plus password invites enabled for external stakeholders.               |
+| **Outbound Email**       | **Google Workspace SMTP Relay**        | Reliable delivery of user invitations, password resets, and automated dashboard subscriptions.                                                 |
+| **Secrets Management**   | **GCP Secret Manager**                 | Securely injects database passwords, encryption keys, and client secrets directly into container environment variables.                        |
+| **IaC & Automation**     | **Terraform + GitHub Actions**         | Fully reproducible infrastructure; automated `terraform plan` on Pull Requests and `terraform apply` via release tagging.                      |
+| **Local Development**    | **VS Code Dev Containers**             | Cross-platform consistency across Linux, macOS, and Windows matching `compilerla` Dev Container standards.                                     |

docs/guides/deployment-production.md

@@ -0,0 +1,9 @@
+# Deploying to Production
+
+This guide explains how to deploy the dashboards to production.
+
+## Steps
+
+1. Run terraform plan
+2. Review changes
+3. Run terraform apply

docs/index.md

@@ -0,0 +1,12 @@
+# Dashboards
+
+Documentation for Compiler's analytics and business intelligence workspace.
+
+This project provides the infrastructure and services for setting up Metabase and GCP resources.
+
+## Getting Started
+
+- [Tutorials](tutorials/) - Learn how to use the system.
+- [Guides](guides/) - Solve specific tasks.
+- [Reference](reference/) - Look up details.
+- [Explanation](explanation/) - Understand the architecture.

docs/reference/infrastructure.md

@@ -0,0 +1,8 @@
+# Infrastructure Reference
+
+This section contains technical details about the infrastructure.
+
+## GCP Resources
+
+- BigQuery Dataset: `dashboards_data`
+- Cloud Run Service: `metabase-api`

docs/reference/scripts.md

@@ -0,0 +1,15 @@
+# Helper Scripts
+
+The following helper scripts are available to run inside the dev container:
+
+- **Wait for Metabase to initialize**:
+
+  ```bash
+  ./bin/wait-for-metabase.sh
+  ```
+
+- **Connect to the local Postgres metadata database**:
+
+  ```bash
+  ./bin/psql.sh
+  ```