docs: publish a documentation site on GitHub Pages

.github/workflows/docs.yml

@@ -0,0 +1,57 @@
+name: Docs
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - doc/**
+      - tool/docsite/**
+      - .github/workflows/docs.yml
+      - MODULE.bazel
+  pull_request:
+    paths:
+      - doc/**
+      - tool/docsite/**
+      - .github/workflows/docs.yml
+      - MODULE.bazel
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build site
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup
+
+      - name: Build site
+        run: make docs-build
+
+      - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
+        with:
+          path: tool/docsite/build
+
+  deploy:
+    name: Deploy to GitHub Pages
+    if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }}
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    # Never cancel a deploy in progress; a newer run queues behind it.
+    concurrency:
+      group: pages
+      cancel-in-progress: false
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1

.gitignore

@@ -23,3 +23,7 @@ bin/
 
 # Make completion cache
 .make_targets_cache
+
+# Documentation site build output
+/tool/docsite/build/
+/tool/docsite/hooks/__pycache__/

.yamlfmt

@@ -3,3 +3,5 @@ formatter:
   indent: 2
   retain_line_breaks_single: true
   include_document_start: false
+exclude:
+  - tool/docsite/build

MODULE.bazel

@@ -73,3 +73,19 @@ use_repo(
     "org_uber_go_yarpc",
     "org_uber_go_zap",
 )
+
+# Python toolchain and PyPI packages for the documentation site (//tool/docsite).
+# Its targets pin this version, so the default toolchain other Python targets use
+# is unchanged. Regenerate the lock with: bazel run //tool/docsite:requirements.update
+DOCSITE_PYTHON_VERSION = "3.13"
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(python_version = DOCSITE_PYTHON_VERSION)
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "docsite_pip",
+    python_version = DOCSITE_PYTHON_VERSION,
+    requirements_lock = "//tool/docsite:requirements_lock.txt",
+)
+use_repo(pip, "docsite_pip")

Makefile

@@ -148,7 +148,7 @@ define assert_clean
 	fi
 endef
 
-.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto demo-requests deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache test-race tidy tidy-bazel tidy-go help
+.PHONY: build build-all-linux build-runway-linux build-submitqueue-gateway-client build-submitqueue-gateway-linux build-submitqueue-gateway-server build-submitqueue-orchestrator-linux build-stovepipe-linux build-stovepipe-linux-debug check-gazelle check-mocks check-tidy clean clean-proto demo-requests deps docs-build docs-serve e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-binary lint-fmt lint-license local-init-runway-queue-schema local-init-stovepipe-schemas local-runway-start local-runway-stop local-submitqueue-stop local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-debug-start local-stovepipe-logs local-stovepipe-start local-stovepipe-stop mocks proto query-deps query-targets run-client-runway run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe run-queue-admin test test-no-cache test-race tidy tidy-bazel tidy-go help
 
 
 build: ## Build all services and examples
@@ -263,6 +263,12 @@ demo-requests: ## Create N changes, enqueue each as it is created, and watch (PR
 deps: tidy-go ## Download and tidy Go dependencies
 	@echo "Dependencies installed!"
 
+docs-build: ## Build the documentation site into tool/docsite/build (fails on broken links)
+	@$(BAZEL) run //tool/docsite:mkdocs -- build --strict --config-file $(CURDIR)/tool/docsite/mkdocs.yml --site-dir $(CURDIR)/tool/docsite/build
+
+docs-serve: ## Serve the documentation site with live reload on a free localhost port (DOCS_PORT=8000 to pin one)
+	@$(BAZEL) run //tool/docsite:mkdocs -- serve --config-file $(CURDIR)/tool/docsite/mkdocs.yml $(if $(DOCS_PORT),--dev-addr 127.0.0.1:$(DOCS_PORT))
+
 e2e-git-test: ## Run the hermetic git E2E (real merger against a bare repo; no credentials)
 	@echo "Running hermetic git end-to-end tests..."
 	@$(BAZEL) test //test/e2e/submitqueue:go_default_test --test_output=errors \

README.md

@@ -1,6 +1,7 @@
 # SubmitQueue
 
 [![CI](https://github.com/uber/submitqueue/actions/workflows/ci.yml/badge.svg)](https://github.com/uber/submitqueue/actions/workflows/ci.yml)
+[![Docs](https://img.shields.io/badge/docs-uber.github.io%2Fsubmitqueue-blue)](https://uber.github.io/submitqueue/)
 [![Go Version](https://img.shields.io/github/go-mod/go-version/uber/submitqueue)](go.mod)
 [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
 [![Slack](https://img.shields.io/badge/Slack-join%20the%20community-4A154B?logo=slack&logoColor=white)](https://join.slack.com/t/submitqueue/shared_invite/zt-46gkqj682-7zcQphxm2pYqkjDo9lbmYA)

doc/BUILD.bazel

@@ -0,0 +1,8 @@
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**"],
+        exclude = ["BUILD.bazel"],
+    ),
+    visibility = ["//tool/docsite:__pkg__"],
+)

doc/index.md

@@ -0,0 +1,97 @@
+---
+title: Home
+template: home.html
+hide:
+  - navigation
+  - toc
+---
+
+<div class="sq-github-only" markdown>
+
+# SubmitQueue
+
+A high-performance speculative submission queue that keeps your trunk consistently green at scale.
+
+</div>
+
+SubmitQueue does not validate changes one at a time. It speculatively rebases and validates many changes in parallel against predicted future states of HEAD. Changes whose validations pass land automatically. When a validation fails, SubmitQueue isolates the offending change and retries the rest, with no human involved. It is designed for large monorepos and fast-moving teams, where concurrent changes can introduce subtle conflicts and destabilize builds.
+
+## Why SubmitQueue
+
+<div class="grid cards sq-features" markdown>
+
+-   :material-source-branch-check:{ .lg .middle } **Speculative validation**
+
+    ---
+
+    SubmitQueue builds a tree of possible future HEADs and validates the paths most likely to land, in parallel.
+
+    [:octicons-arrow-right-24: Speculation](rfc/submitqueue/speculation.md)
+
+-   :material-shield-check-outline:{ .lg .middle } **Isolates failures**
+
+    ---
+
+    A failing change is isolated and rejected while the rest of its batch carries on to land.
+
+    [:octicons-arrow-right-24: Orchestrator workflow](rfc/submitqueue/workflow.md)
+
+-   :material-puzzle-outline:{ .lg .middle } **Pluggable extensions**
+
+    ---
+
+    Build runners, change providers, storage, queues and scorers are vendor-agnostic interfaces with swappable implementations.
+
+    [:octicons-arrow-right-24: Extension contract](rfc/submitqueue/extension-contract.md)
+
+-   :material-tray-full:{ .lg .middle } **Durable, queue-driven pipeline**
+
+    ---
+
+    Every stage is an idempotent consumer on an at-least-once message queue, with optimistic locking and no distributed transactions.
+
+    [:octicons-arrow-right-24: SQL-based queue](rfc/sql-queue-rfc.md)
+
+</div>
+
+## Components
+
+<div class="grid cards" markdown>
+
+-   :material-call-merge:{ .lg .middle } **SubmitQueue**
+
+    ---
+
+    The gateway and orchestrator that accept, batch, speculate on, build and land changes.
+
+    [:octicons-arrow-right-24: Workflow](rfc/submitqueue/workflow.md)
+
+-   :material-airplane-landing:{ .lg .middle } **Runway**
+
+    ---
+
+    The landing service. It owns VCS operations, conflict checks and merges, on SubmitQueue's behalf.
+
+    [:octicons-arrow-right-24: Workflow](rfc/runway/workflow.md)
+
+-   :material-check-decagram-outline:{ .lg .middle } **Stovepipe**
+
+    ---
+
+    The post-land pipeline. It validates landed commits and tracks the last green revision.
+
+    [:octicons-arrow-right-24: Workflow](rfc/stovepipe/workflow.md)
+
+</div>
+
+## Try it in a minute
+
+You need only Docker. No repository, account or token is required.
+
+```bash
+make local-submitqueue-start   # Gateway + Orchestrator + Runway + MySQL
+make demo-requests             # create changes, enqueue them, watch them land
+make local-submitqueue-stop
+```
+
+The [Quickstart](howto/QUICKSTART.md) goes from a fake provider to a local git repository to real GitHub pull requests. Questions? Join the [Slack community](https://join.slack.com/t/submitqueue/shared_invite/zt-46gkqj682-7zcQphxm2pYqkjDo9lbmYA).

tool/README.md

@@ -23,6 +23,10 @@ bazel build //...
 
 The Bazel version is controlled by `.bazelversion` at the repository root. Update that file to change the Bazel version used by the wrapper.
 
+## Documentation site
+
+`tool/docsite` holds the MkDocs Material configuration that renders `doc/` as the site published at https://uber.github.io/submitqueue/. The site reads `doc/` in place; `doc/index.md` is its landing page. Links that leave `doc/` (code, package READMEs, `AGENTS.md`) are rewritten to GitHub by `hooks/repo_links.py`, so the strict build still fails on a broken link between pages. The nav is generated from the directory tree; `hooks/nav_titles.py` maps directory names to section titles (`howto` → Guides, `rfc` → Design (RFCs)), and `overrides/home.html` renders the landing-page hero. MkDocs runs under Bazel with a hermetic Python toolchain and locked dependencies: `make docs-serve` previews the site locally, `make docs-build` runs the strict build into `tool/docsite/build`, and `//tool/docsite:site_test` runs that strict build as part of `make test`. To change dependency versions, edit `requirements.txt` and run `bazel run //tool/docsite:requirements.update` to regenerate `requirements_lock.txt`. `.github/workflows/docs.yml` builds the site on pull requests and deploys it from `main`.
+
 ## Git sandbox
 
 `tool/gitsandbox` creates the bare repository that `make local-submitqueue-start PROVIDER=git` merges into. It runs before the stack starts, because Runway clones that repository at boot and fails if the target does not already exist. The result is one seed commit on the target branch. Running it again leaves an existing repository unchanged, so a restart keeps commits that earlier runs landed. The Makefile invokes it; `bazel run //tool/gitsandbox -- -sandbox-dir <dir>` is the direct form.

tool/docsite/BUILD.bazel

@@ -0,0 +1,53 @@
+load("@docsite_pip//:requirements.bzl", "requirement")
+load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+
+DOCSITE_PYTHON_VERSION = "3.13"
+
+compile_pip_requirements(
+    name = "requirements",
+    src = "requirements.txt",
+    python_version = DOCSITE_PYTHON_VERSION,
+    requirements_txt = "requirements_lock.txt",
+    # The generated lock-freshness test downloads from PyPI; keep it out of //... runs.
+    tags = ["manual"],
+)
+
+filegroup(
+    name = "site_sources",
+    srcs = [
+        "mkdocs.yml",
+        "//doc:all_files",
+    ] + glob(
+        [
+            "hooks/*.py",
+            "overrides/**",
+        ],
+    ),
+)
+
+py_binary(
+    name = "mkdocs",
+    srcs = ["mkdocs_main.py"],
+    legacy_create_init = 0,
+    main = "mkdocs_main.py",
+    python_version = DOCSITE_PYTHON_VERSION,
+    deps = [
+        requirement("mkdocs"),
+        requirement("mkdocs-material"),
+    ],
+)
+
+py_test(
+    name = "site_test",
+    size = "small",
+    srcs = ["site_test.py"],
+    data = [":site_sources"],
+    legacy_create_init = 0,
+    main = "site_test.py",
+    python_version = DOCSITE_PYTHON_VERSION,
+    deps = [
+        requirement("mkdocs"),
+        requirement("mkdocs-material"),
+    ],
+)

tool/docsite/hooks/nav_titles.py

@@ -0,0 +1,57 @@
+"""Give navigation sections readable titles.
+
+The nav is generated from the directory tree so new docs appear without config
+changes; MkDocs titles a section by capitalizing its directory name, which
+yields "Howto" and "Rfc". This maps directory names to display titles instead.
+"""
+
+from mkdocs.structure.nav import Section
+from mkdocs.structure.pages import Page
+
+SECTION_TITLES = {
+    "howto": "Guides",
+    "rfc": "Design (RFCs)",
+    "runway": "Runway",
+    "steps": "Stages",
+    "stovepipe": "Stovepipe",
+    "submitqueue": "SubmitQueue",
+}
+
+# Pages listed here sort first within their section, in this order.
+LEADING_PAGES = ["howto/QUICKSTART.md"]
+
+
+def on_nav(nav, config, files):
+    _retitle_sections(nav.items)
+    _relink_previous_and_next_pages(nav)
+    return nav
+
+
+def _retitle_sections(items):
+    for item in items:
+        if not isinstance(item, Section):
+            continue
+        item.title = SECTION_TITLES.get(item.title.lower(), item.title)
+        item.children.sort(key=_leading_page_rank)
+        _retitle_sections(item.children)
+
+
+def _relink_previous_and_next_pages(nav):
+    # MkDocs links previous/next pages before on_nav runs, so a reorder must redo them.
+    nav.pages = list(_pages_in_order(nav.items))
+    for i, page in enumerate(nav.pages):
+        page.previous_page = nav.pages[i - 1] if i > 0 else None
+        page.next_page = nav.pages[i + 1] if i + 1 < len(nav.pages) else None
+
+
+def _pages_in_order(items):
+    for item in items:
+        if isinstance(item, Section):
+            yield from _pages_in_order(item.children)
+        elif isinstance(item, Page):
+            yield item
+
+
+def _leading_page_rank(item):
+    src = getattr(getattr(item, "file", None), "src_uri", None)
+    return LEADING_PAGES.index(src) if src in LEADING_PAGES else len(LEADING_PAGES)

tool/docsite/hooks/repo_links.py

@@ -0,0 +1,42 @@
+"""Rewrite doc links that leave docs_dir into links to the GitHub source tree.
+
+The docs link to code and package READMEs elsewhere in the repository. Those
+targets are not part of the site, so strict mode would reject them; pointing
+them at GitHub keeps strict mode on for links between pages.
+"""
+
+import os
+import re
+
+REPO_BLOB_URL = "https://github.com/uber/submitqueue/blob/main/"
+REPO_TREE_URL = "https://github.com/uber/submitqueue/tree/main/"
+
+INLINE_LINK = re.compile(r"(\]\()([^)\s]+)(\))")
+REFERENCE_LINK = re.compile(r"^(\s*\[[^\]]+\]:\s*)(\S+)(.*)$", re.MULTILINE)
+
+
+def on_page_markdown(markdown, page, config, files):
+    docs_dir = os.path.abspath(config["docs_dir"])
+    repo_root = os.path.dirname(docs_dir)
+    page_dir = os.path.dirname(os.path.join(docs_dir, page.file.src_path))
+
+    def rewrite_target(target):
+        if re.match(r"^[a-z][a-z0-9+.-]*:", target) or target.startswith(("#", "/")):
+            return target
+        path, sep, anchor = target.partition("#")
+        resolved = os.path.normpath(os.path.join(page_dir, path))
+        if resolved == docs_dir or resolved.startswith(docs_dir + os.sep):
+            return target
+        if not resolved.startswith(repo_root + os.sep):
+            return target
+        rel = os.path.relpath(resolved, repo_root).replace(os.sep, "/")
+        base = REPO_TREE_URL if os.path.isdir(resolved) else REPO_BLOB_URL
+        return base + rel + sep + anchor
+
+    def rewrite_prose(text):
+        text = INLINE_LINK.sub(lambda m: m.group(1) + rewrite_target(m.group(2)) + m.group(3), text)
+        return REFERENCE_LINK.sub(lambda m: m.group(1) + rewrite_target(m.group(2)) + m.group(3), text)
+
+    # Even-indexed chunks are prose, odd-indexed chunks are fenced code.
+    chunks = re.split(r"(^\s*(?:```|~~~).*?^\s*(?:```|~~~)[^\n]*$)", markdown, flags=re.MULTILINE | re.DOTALL)
+    return "".join(rewrite_prose(c) if i % 2 == 0 else c for i, c in enumerate(chunks))