diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 85f9616..dba099f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,10 +2,15 @@ FROM python:3.13-bookworm ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ - USER=compiler + USER=compiler \ + USER_UID=1000 \ + USER_GID=1000 # Install system dependencies and patch OS packages to reduce known vulnerabilities -RUN apt-get update && apt-get upgrade -y --no-install-recommends && apt-get install -y --no-install-recommends \ +# using the archives and lists cache +RUN --mount=type=cache,id=apt-archives,sharing=locked,target=/var/cache/apt/archives \ + --mount=type=cache,id=apt-lists,sharing=locked,target=/var/lib/apt/lists \ + apt-get update && apt-get upgrade -y --no-install-recommends && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ git \ @@ -43,8 +48,20 @@ USER $USER WORKDIR /home/$USER/src # Update PATH for local pip installs -ENV PATH="$PATH:/home/$USER/.local/bin" +ENV PATH="$PATH:/home/$USER/.local/bin" \ + # update env for local pip installs + # see https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE + # since all `pip install` commands are in the context of $USER + # $PYTHONUSERBASE is the location used by default + PYTHONUSERBASE="/home/${USER}/.local" \ + # where to store the pip cache (use the default) + # https://pip.pypa.io/en/stable/cli/pip/#cmdoption-cache-dir + PIP_CACHE_DIR="/home/${USER}/.cache/pip" -# Update pip and install pre-commit -RUN python -m pip install --upgrade pip && \ - python -m pip install --no-cache-dir pre-commit +COPY docs/requirements.txt docs/requirements.txt + +# Update pip and install Python tooling +RUN --mount=type=cache,id=pipcache,target=${PIP_CACHE_DIR},uid=${USER_UID},gid=${USER_GID} \ + python -m pip install --user --upgrade pip && \ + python -m pip install --user pre-commit && \ + python -m pip install --user -r docs/requirements.txt diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bbc24d5..e0ef013 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "name": "compilerla/dashboards", "dockerComposeFile": ["../compose.yml"], "service": "dev", - "runServices": ["dev"], + "runServices": ["dev", "docs"], "workspaceFolder": "/home/compiler/src", "postCreateCommand": ["/bin/bash", ".devcontainer/postCreate.sh"], "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"], diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml new file mode 100644 index 0000000..5c9f232 --- /dev/null +++ b/.github/workflows/mkdocs.yml @@ -0,0 +1,99 @@ +name: Publish docs +on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - ".github/workflows/mkdocs.yml" + - "docs/**" + - "mkdocs.yml" + push: + branches: + - main + paths: + - ".github/workflows/mkdocs.yml" + - "docs/**" + - "mkdocs.yml" + +jobs: + docs-preview: + name: Publish docs preview + runs-on: ubuntu-latest + if: github.event.pull_request + permissions: + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: "refs/pull/${{ github.event.number }}/merge" + + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: "docs/requirements.txt" + + - name: Build MkDocs website + run: | + pip install -r docs/requirements.txt + mkdocs build + + - name: Install Netlify CLI + run: npm install --location=global netlify-cli@17.x.x + + - name: Deploy Preview to Netlify + run: | + netlify deploy \ + --alias="${GITHUB_REPOSITORY#*/}-${{ github.event.number }}" \ + --auth=${{ secrets.NETLIFY_AUTH_TOKEN }} \ + --dir="site" \ + --site=${{ vars.NETLIFY_PREVIEW_APP_SITE_ID }} + + - name: Find existing comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + id: find-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: "Preview url: https://" + + - name: Add Netlify link PR comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + if: steps.find-comment.outputs.comment-id == '' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const hostnameSuffix = "compiler-previews.netlify.app" + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `Preview url: https://${context.repo.repo}-${{ github.event.number }}--${hostnameSuffix}`, + }) + + docs: + name: Publish docs + runs-on: ubuntu-latest + if: github.event.pull_request == null && github.event.push.branch == 'main' + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-python@46b300bd3776ad20ef3e26060101221095800131 # v6.0.0 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: "docs/requirements.txt" + + - name: Install MkDocs requirements + run: | + pip install -r docs/requirements.txt + + - name: Deploy MkDocs website + run: | + mkdocs gh-deploy --force diff --git a/AGENTS.md b/AGENTS.md index 9c0dc2d..d30484a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,8 @@ - **Terraform Execution**: ALWAYS run terraform commands via `exec` against the existing, running `dev` compose service. Do not run them directly on the host and do not run them in a new ephemeral `dev` service. This is where GCP credentials will be configured. - **Verification**: Run `terraform fmt` and `terraform validate` after every terraform file addition, modification, etc. - **Pre-commit**: Hooks are installed automatically via `.devcontainer/postCreate.sh`. Run `pre-commit run --all --all-files` after making file modifications. +- **GitHub Actions**: ALWAYS use the latest full commit SHA reference for external steps, with a comment of the current version number. +- **Docs**: When making changes, the docs should be checked and updated accordingly if necessary. ## Constraints & Gotchas diff --git a/README.md b/README.md index 2fdf67b..a76284b 100644 --- a/README.md +++ b/README.md @@ -2,81 +2,10 @@ Compiler's analytics and business intelligence workspace, powered by open-source [Metabase](https://www.metabase.com/). ---- +## Documentation -## Local Development (VS Code Dev Containers) +For detailed instructions on local development, architecture, and repository structure, please visit our documentation site: -This repository follows Compiler's cross-platform [Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers) standard to ensure consistent development environments across macOS, Linux, and Windows. +[docs.compiler.la/dashboards](https://docs.compiler.la/dashboards) -### Prerequisites - -1. [Docker Desktop](https://www.docker.com/products/docker-desktop/) or Docker Engine running locally. -2. [Visual Studio Code](https://code.visualstudio.com/). -3. VS Code [Dev Containers Extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). - -### Getting Started - -1. Copy the sample environment file: - - ```bash - cp .env.example .env - ``` - -2. Open this folder in VS Code. -3. When prompted, click **"Reopen in Container"** (or press `F1` / `Ctrl+Shift+P` / `Cmd+Shift+P` and run `Dev Containers: Reopen in Container`). -4. VS Code will build the dev container and launch all three services: - - **`dev`**: Your development shell containing `terraform`, `gcloud`, `pre-commit`, and Postgres client tools. - - **`postgres`**: Local PostgreSQL 16 database storing Metabase application state. - - **`metabase`**: Official Metabase application available at [http://localhost:3000](http://localhost:3000). - -### Helper Scripts (`bin/`) - -The following helper scripts are available to run inside the dev container: - -- **Wait for Metabase to initialize**: - - ```bash - ./bin/wait-for-metabase.sh - ``` - -- **Connect to the local Postgres metadata database**: - - ```bash - ./bin/psql.sh - ``` - ---- - -## Architecture Overview - -Production is deployed on **Google Cloud Platform (GCP)**: - -- **Cloud Run**: Runs the Metabase container image (`metabase/metabase:vX.Y.Z`) with minimum 1 instance to support background tasks and scheduled pulses. -- **Cloud SQL (PostgreSQL)**: Stores Metabase application metadata (dashboards, questions, users, permissions). -- **BigQuery**: Analytics warehouse in project `data-warehouse-492522`, queried via native IAM Service Account credentials. -- **Cloudflare**: DNS management and SSL termination for `dashboards.compiler.la`. - -For the complete technical specification, see [plan.md](plan.md). - ---- - -## Repository Structure - -```text -. -├── .devcontainer/ -│ ├── devcontainer.json # VS Code dev container definition -│ ├── Dockerfile # Dev container image (Terraform, gcloud, pre-commit) -│ └── postCreate.sh # Container initialization & hook setup -├── .github/ -│ └── workflows/ # GitHub Actions for Terraform automation -├── bin/ # Helper bash scripts for inside the dev container -│ ├── psql.sh # Connect to local PostgreSQL -│ └── wait-for-metabase.sh # Health check waiter -├── compose.yml # Dev container service + local Metabase & Postgres -├── .env.example # Sample local environment variables -├── .pre-commit-config.yaml # Conventional commits & file hygiene hooks -├── plan.md # Full architectural design & implementation plan -├── terraform/ # Infrastructure as Code (GCP Cloud Run & Cloud SQL) -└── README.md -``` +_Note: The documentation site is available at locally when running the `docs` service via `compose.yml`._ diff --git a/compose.yml b/compose.yml index 9c6b8b6..b40b076 100644 --- a/compose.yml +++ b/compose.yml @@ -20,7 +20,14 @@ services: condition: "service_healthy" metabase: condition: "service_started" - + docs: + image: "compilerla/dashboards:dev" + entrypoint: mkdocs + command: serve --livereload --dev-addr "0.0.0.0:8001" + ports: + - "8001:8001" + volumes: + - "./:/home/compiler/src" metabase: image: "metabase/metabase:v0.65.x" container_name: "metabase" @@ -39,7 +46,6 @@ services: postgres: condition: "service_healthy" restart: "unless-stopped" - postgres: image: "postgres:18-alpine" container_name: "postgres" @@ -58,3 +64,4 @@ services: volumes: postgres_data: + driver: local diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md new file mode 100644 index 0000000..612c44d --- /dev/null +++ b/docs/explanation/architecture.md @@ -0,0 +1,65 @@ +# Project Architecture + +This section explains the reasoning behind our design choices. + +## Architecture Overview + +```mermaid +flowchart TD + subgraph Users & Ingress + U_Internal["Internal Users (Google Workspace SSO)"] + U_External["External Users (Username / Password)"] + CF["Cloudflare DNS & Proxy (dashboards.compiler.la)"] + end + + subgraph GCP ["Google Cloud Platform (Project: data-warehouse-492522 | Region: us-west1)"] + subgraph CloudRun ["Cloud Run (Managed Container Runtime)"] + MB["Metabase OSS Container
(metabase/metabase:vX.Y.Z)
Min Instances: 1 | 2 GiB RAM"] + end + + subgraph Storage_State ["Application State & Secrets"] + CSQL[("Cloud SQL PostgreSQL
(Application Metadata & Dashboards)")] + SM["Secret Manager
(DB Pass, Encryption Key, OAuth Secret)"] + end + + subgraph Analytics ["Analytics Warehouse"] + BQ[("Google BigQuery (data-warehouse-492522)")] + end + + subgraph IAM_Service ["Security & Identity"] + SA["Cloud Run Service Account
(roles/bigquery.jobUser
roles/bigquery.dataViewer)"] + end + end + + subgraph Outbound ["Notifications"] + GW_SMTP["Google Workspace SMTP Relay
(Invitations, Alerts, Password Resets)"] + end + + U_Internal --> CF + U_External --> CF + CF -->|HTTPS| MB + + MB -->|Cloud SQL Connection Unix Socket| CSQL + MB -.->|Fetch Secrets on Boot| SM + MB -->|Native IAM Auth / Zero Keys| BQ + SA -.->|Attached Identity| MB + + MB -->|Port 587 TLS| GW_SMTP +``` + +## Key Architecture Decisions + +| Component | Selected Configuration | Rationale | +| :----------------------- | :------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------- | +| **GCP Project** | `data-warehouse-492522` | Host project containing BigQuery analytics warehouse and infrastructure. | +| **GCP Region** | `us-west1` (Oregon) | Optimal low-latency to California; standard Tier 1 GCP pricing (matches `us-central1` costs). | +| **Domain & DNS** | `dashboards.compiler.la` (Cloudflare) | Free managed SSL certificates; CNAME pointing to Cloud Run; proxy/WAF protection. | +| **Container Runtime** | **GCP Cloud Run** | Zero Kubernetes overhead; fully managed container lifecycle; integrated Cloud SQL proxy; native HTTPS. | +| **Instance Policy** | **Min Instances: 1** | Prevents cold-start delays and ensures Metabase background syncs, alerts, and scheduled pulses run continuously. | +| **Application Database** | **Cloud SQL PostgreSQL** | Production-ready state store for Metabase (dashboards, queries, users, permissions). Never use the default embedded H2 database in production. | +| **Analytics Warehouse** | **Google BigQuery** | Native driver built into Metabase. Authenticated via the Cloud Run attached Service Account (no JSON key rotation needed). | +| **Authentication** | **Dual Auth: Google OAuth + Password** | Google Workspace OAuth restricted to `@compiler.la` for internal staff, plus password invites enabled for external stakeholders. | +| **Outbound Email** | **Google Workspace SMTP Relay** | Reliable delivery of user invitations, password resets, and automated dashboard subscriptions. | +| **Secrets Management** | **GCP Secret Manager** | Securely injects database passwords, encryption keys, and client secrets directly into container environment variables. | +| **IaC & Automation** | **Terraform + GitHub Actions** | Fully reproducible infrastructure; automated `terraform plan` on Pull Requests and `terraform apply` via release tagging. | +| **Local Development** | **VS Code Dev Containers** | Cross-platform consistency across Linux, macOS, and Windows matching `compilerla` Dev Container standards. | diff --git a/docs/guides/deployment-production.md b/docs/guides/deployment-production.md new file mode 100644 index 0000000..f97c14d --- /dev/null +++ b/docs/guides/deployment-production.md @@ -0,0 +1,9 @@ +# Deploying to Production + +This guide explains how to deploy the dashboards to production. + +## Steps + +1. Run terraform plan +2. Review changes +3. Run terraform apply diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..3755e37 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,12 @@ +# Dashboards + +Documentation for Compiler's analytics and business intelligence workspace. + +This project provides the infrastructure and services for setting up Metabase and GCP resources. + +## Getting Started + +- [Tutorials](tutorials/) - Learn how to use the system. +- [Guides](guides/) - Solve specific tasks. +- [Reference](reference/) - Look up details. +- [Explanation](explanation/) - Understand the architecture. diff --git a/docs/reference/infrastructure.md b/docs/reference/infrastructure.md new file mode 100644 index 0000000..935f9b1 --- /dev/null +++ b/docs/reference/infrastructure.md @@ -0,0 +1,8 @@ +# Infrastructure Reference + +This section contains technical details about the infrastructure. + +## GCP Resources + +- BigQuery Dataset: `dashboards_data` +- Cloud Run Service: `metabase-api` diff --git a/docs/reference/scripts.md b/docs/reference/scripts.md new file mode 100644 index 0000000..987fa41 --- /dev/null +++ b/docs/reference/scripts.md @@ -0,0 +1,15 @@ +# Helper Scripts + +The following helper scripts are available to run inside the dev container: + +- **Wait for Metabase to initialize**: + + ```bash + ./bin/wait-for-metabase.sh + ``` + +- **Connect to the local Postgres metadata database**: + + ```bash + ./bin/psql.sh + ``` diff --git a/docs/reference/structure.md b/docs/reference/structure.md new file mode 100644 index 0000000..f9cf5a7 --- /dev/null +++ b/docs/reference/structure.md @@ -0,0 +1,20 @@ +# Repository Structure + +```text +. +├── .devcontainer/ +│ ├── devcontainer.json # VS Code dev container definition +│ ├── Dockerfile # Dev container image (Terraform, gcloud, pre-commit) +│ └── postCreate.sh # Container initialization & hook setup +├── .github/ +│ └── workflows/ # GitHub Actions for Terraform automation +├── bin/ # Helper bash scripts for inside the dev container +│ ├── psql.sh # Connect to local PostgreSQL +│ └── wait-for-metabase.sh # Health check waiter +├── compose.yml # Dev container service + local Metabase & Postgres +├── .env.example # Sample local environment variables +├── .pre-commit-config.yaml # Conventional commits & file hygiene hooks +├── plan.md # Full architectural design & implementation plan +├── terraform/ # Infrastructure as Code (GCP Cloud Run & Cloud SQL) +└── README.md +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..ad258c3 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,6 @@ +mdx_truly_sane_lists +mkdocs==1.6.1 +mkdocs-awesome-pages-plugin +mkdocs-macros-plugin +mkdocs-material==9.7.7 +mkdocs-redirects diff --git a/docs/tutorials/getting-started.md b/docs/tutorials/getting-started.md new file mode 100644 index 0000000..491f784 --- /dev/null +++ b/docs/tutorials/getting-started.md @@ -0,0 +1,32 @@ +# Getting Started with Dashboards + +This is a tutorial on how to set up your local development environment. + +## Local Development (VS Code Dev Containers) + +This repository follows Compiler's cross-platform [Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers) standard to ensure consistent development environments across macOS, Linux, and Windows. + +!!! note + + Read our [blog post on Dev Containers](https://compiler.la/blog/2024/devcontainer-platform-agnostic-team/) to learn more about Compiler's cross-platform development philosophy. + +### Prerequisites + +1. Docker and Docker Compose running locally (check out [Podman](https://podman.io/) for a cross-platform OSS management environment). +2. [Visual Studio Code](https://code.visualstudio.com/). +3. VS Code [Dev Containers Extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). + +### Getting Started + +1. Copy the sample environment file: + + ```bash + cp .env.example .env + ``` + +2. Open this folder in VS Code. +3. When prompted, click **"Reopen in Container"** (or press `F1` / `Ctrl+Shift+P` / `Cmd+Shift+P` and run `Dev Containers: Reopen in Container`). +4. VS Code will build the dev container and launch all three services: + - **`dev`**: Your development shell containing `terraform`, `gcloud`, `pre-commit`, and Postgres client tools. + - **`postgres`**: Local PostgreSQL 16 database storing Metabase application state. + - **`metabase`**: Official Metabase application available at [http://localhost:3000](http://localhost:3000). diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..d8d5aef --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,61 @@ +site_name: "Compiler Dashboards: Documentation" +repo_url: https://github.com/compilerla/dashboards +edit_uri: edit/main/docs +site_url: https://docs.compiler.la/dashboards + +theme: + name: material + features: + - content.code.copy + - navigation.expand + - navigation.tabs + - toc.integrate + favicon: https://compiler.la/favicon.ico + logo: https://avatars.githubusercontent.com/u/13874321?s=100&v=4 + palette: + primary: green + accent: deep purple + +extra: + analytics: + provider: google + property: G-VR1QBF67SK + +markdown_extensions: + - admonition + - attr_list + - md_in_html + - codehilite: + linenums: true + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.inlinehilite + - pymdownx.snippets: + base_path: !relative + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.tabbed: + alternate_style: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.smartsymbols + - meta + - toc: + permalink: " ¶" + - smarty + +plugins: + - awesome-pages + - redirects: + redirect_maps: + - search + +extra_css: + - https://use.fontawesome.com/releases/v6.1.2/css/all.css