diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..c3372fa --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,40 @@ +name: Docs + +# Checks Markdown links and builds the manual. +on: + pull_request: + branches: + - staging + - main + paths: + - "**.md" + - "docs/**" + - "book.toml" + - "scripts/build-book.sh" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + # Local files and anchors only, so an external site being down can't fail a PR. + - name: Check Markdown links + uses: lycheeverse/lychee-action@v2 + with: + args: --offline --include-fragments --no-progress README.md CONTRIBUTING.md SECURITY.md sdk/README.md "docs/**/*.md" + + - name: Build + run: make book + + # Catches links that resolve on GitHub but not in the book, such as to files outside docs/. + - name: Check book links + uses: lycheeverse/lychee-action@v2 + with: + args: --offline --include-fragments --no-progress --root-dir "${{ github.workspace }}/book" "book/**/*.html" diff --git a/.gitignore b/.gitignore index eca77a7..2ab87d2 100644 --- a/.gitignore +++ b/.gitignore @@ -83,8 +83,11 @@ instance/ # Scrapy stuff: .scrapy -# Sphinx documentation -docs/_build/ +# mdBook output and tools, see scripts/build-book.sh +/book/ +/.mdbook/ +/mermaid.min.js +/mermaid-init.js # PyBuilder target/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cbbeec0..9183792 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ ## Reporting security issues > [!CAUTION] -> Don't open a public issue for a vulnerability. Email **** with a description, the affected versions and steps to reproduce. +> Don't open a public issue for a vulnerability. See [SECURITY.md](SECURITY.md). ## Development setup @@ -88,7 +88,7 @@ Each service can also run on its own: | Celery worker and beat | `celery -A publisher.tasks.celery_app:celery_app worker` / `beat` | | CLI | `python -m publisher --help` (or `./publisher.sh`) | -`python -m tools.client` drives the gRPC and REST flows by hand, see [tools/README.md](tools/README.md). +`python -m tools.client` drives the gRPC and REST flows by hand, see [docs/developers/reference-client.md](docs/developers/reference-client.md). ### Make targets @@ -104,6 +104,8 @@ Each service can also run on its own: | `make test` | Run pytest | | `make coverage` | Run pytest and list the lines no test runs | | `make docs` | Regenerate `docs/openapi.json` from the REST app | +| `make book` | Build the manual in `docs/` into `book/` | +| `make book-serve` | Preview the manual on , rebuilding on changes | | `make test-dialects` | Migrate and query SQLite, SQLCipher, Postgres, MySQL and MariaDB (podman) | | `make test-e2e` | Install, update and uninstall in a systemd container (podman) | | `make check` | Run every pre-commit hook on all files | @@ -173,8 +175,8 @@ Common types are `feat`, `fix`, `refactor`, `test`, `docs`, `build`, `ci` and `c - [ ] The PR targets `staging` and does one thing. - [ ] `make check` and `make test` pass locally. -- [ ] New behaviour and bug fixes come with tests. A bug fix has a test that fails without it. -- [ ] Docs, `template.env` and the README are updated if behaviour, settings or commands changed. +- [ ] New behavior and bug fixes come with tests. A bug fix has a test that fails without it. +- [ ] The manual in `docs/` and `template.env` are updated if behavior, settings or commands changed. - [ ] Schema changes include a migration, and a commit message with `[migrate]`. - [ ] The description says what changed, why, and how it was tested. Link the issue with `Closes #123`. - [ ] Anything operators must do on deploy is called out. @@ -213,7 +215,7 @@ The gRPC service handles the account side: storing OAuth2 and PNBA tokens, revok The hooks enforce formatting, lint, types and commit messages. These rules are checked in review: -- **Functions by default.** Modules group related behaviour. Use a class only for real state (a cache, a connection) or when a framework needs one (ORM models, structs, interceptors, servicers, config sections). +- **Functions by default.** Modules group related behavior. Use a class only for real state (a cache, a connection) or when a framework needs one (ORM models, structs, interceptors, servicers, config sections). - **Dependencies are arguments.** Domain and model functions take the database `session` first, then anything else they need, such as the acting credential. - **The entry point owns the transaction.** A route, gRPC handler, task or CLI command opens the session; code below it never opens a session or commits. - **No transaction stays open while an adapter runs,** since a call can take a minute. Publishing is the one exception (see above). @@ -251,7 +253,7 @@ Tests run against in-memory SQLite. `tests/conftest.py` sets the environment bef ### Writing tests - Put a test where its module lives: `publisher/api/rest/v1/stats.py` is tested in `tests/api/rest/test_stats.py`. -- Name tests after the behaviour, such as `test_list_skips_disabled_items`, not the function. +- Name tests after the behavior, such as `test_list_skips_disabled_items`, not the function. - Test through the public interface (a route, an RPC, a domain function). Patch only what crosses the process boundary, such as adapter IPC or outbound HTTP. ### Fixtures and helpers @@ -291,7 +293,7 @@ make build test PYTHON=.venv312/bin/python ### Add a setting 1. Add the field to its section in `publisher/config.py` and read it in that section's `load`. -2. Add it, commented, to `template.env` and to the Configuration section of the README. +2. Add it, commented, to `template.env` and to [docs/configuration/settings.md](docs/configuration/settings.md); a test fails until both list it. 3. Check validation with `python -m publisher config check`, and add a test in `tests/test_config.py`. ### Change the database schema @@ -306,14 +308,14 @@ make build test PYTHON=.venv312/bin/python 1. Add the route to the matching module in `publisher/api/rest/v1/` (or a new module with its own `APIRouter`, registered in `routes.py`). 2. Put request and response models in `schemas.py`. Use `Depends(get_db)` for the session, and `Security(authorize, scopes=[...])` if it needs a login. A new scope goes in `Scope` in `publisher/models/credential.py`. 3. Map domain errors to `ApiError` (`errors.py`) with a message a client can show, and put ids and other detail in its `log=`. -4. Test it in `tests/api/rest/`, describe it in its docstring and run `make docs`. Update [docs/rest.md](docs/rest.md) only for what the reference doesn't show. +4. Test it in `tests/api/rest/`, describe it in its docstring and run `make docs`. Update [docs/reference/rest-api.md](docs/reference/rest-api.md) only for what the reference doesn't show. ### Change the gRPC API 1. Edit `protos/v3/publisher.proto` and run `make protos`. 2. Add a module per RPC in `publisher/api/grpc/v3/` and a thin method on the servicer that calls it. 3. Raise `grpc_interceptor` exceptions such as `InvalidArgument` or `Unauthenticated`; `ErrorInterceptor` turns them into status codes and anything else into `INTERNAL`. -4. Test it in `tests/api/grpc/` and update [docs/grpc.md](docs/grpc.md). +4. Test it in `tests/api/grpc/` and update [docs/reference/grpc-api.md](docs/reference/grpc-api.md). > [!WARNING] > Deployed clients depend on the proto: only add fields. @@ -328,13 +330,20 @@ Add it under `publisher/tasks/`. Keep the explicit `tasks.*` name so queued mess ### Platform adapters -Adapters live in their own repositories and are built with the SDK in `sdk/` ([sdk/README.md](sdk/README.md)). Their CI runs `.github/workflows/adapter.yml` at the SDK tag they pin, so a change to it reaches an adapter only when it moves its pin. Install one locally with `python -m publisher platforms add `; on a server use `./publisher.sh platforms`, which runs as the service user. See [docs/platforms.md](docs/platforms.md). +Adapters live in their own repositories and are built with the SDK in `sdk/` ([sdk/README.md](sdk/README.md)). Their CI runs `.github/workflows/adapter.yml` at the SDK tag they pin, so a change to it reaches an adapter only when it moves its pin. Install one locally with `python -m publisher platforms add `; on a server use `./publisher.sh platforms`, which runs as the service user. See [docs/platforms/manage.md](docs/platforms/manage.md). ## Documentation -- [docs/rest.md](docs/rest.md) (with the generated [docs/openapi.json](docs/openapi.json)), [docs/grpc.md](docs/grpc.md) and [docs/smtp.md](docs/smtp.md) describe each interface. Update them in the same PR as the behaviour. -- The README covers installation, configuration and operations. Keep it in step with `template.env` and the CLI. -- Write plainly and briefly. Use code blocks for commands and tables for options. +The manual in `docs/` is an [mdBook](https://rust-lang.github.io/mdBook/) that reads the same on GitHub. Preview it with `make book-serve`. + +- Update the pages a change affects in the same PR. +- List a new page in `docs/SUMMARY.md`, or the book leaves it out. +- Link pages in `docs/` relatively, and files outside it by their full GitHub URL, since the book only has `docs/`. +- Draw diagrams as `mermaid` code blocks. +- Say "web client" and "public", and use `publisher.example.com`, `example.com` and `+237600000000` in examples. Keep GitHub alerts for what readers must not miss. +- Start a task page with its requirements. End a page that points onward with a `## Next steps` section of one or two linked sentences or a short list; skip it when the book's next page is the only next step. +- Use tables for reference data, numbered steps for procedures, and one heading per symptom for troubleshooting. +- Write plainly and briefly in US English, one paragraph per line, with sentence-case headings and `` placeholders. ## When CI fails diff --git a/INSTALL.md b/INSTALL.md deleted file mode 100644 index 3bb233d..0000000 --- a/INSTALL.md +++ /dev/null @@ -1,270 +0,0 @@ -# Installation Guide - -The [README](README.md#install-on-a-server) covers the one-line installer. This guide covers running several instances, installing by hand, nginx, and every setting. - -## Contents - -- [Running multiple instances](#running-multiple-instances) -- [Manual installation](#manual-installation) -- [Nginx reverse proxy](#nginx-reverse-proxy) -- [Service management](#service-management) -- [The publisher.sh CLI](#the-publishersh-cli) -- [Configuration](#configuration) - -## Running Multiple Instances - -Give each copy its own install directory and instance name: - -```bash -sudo ./install.sh --install-dir /srv/relaysms-acme --instance-name acme -``` - -Its units are namespaced (`relaysms-publisher-acme.target`, `relaysms-publisher-acme-rest.service`, ...), and the `manage.sh` in each install directory manages only its own instance. Re-running `install.sh` remembers the name; a different `--instance-name` is rejected. - -> [!IMPORTANT] -> Set distinct `PORT` and `GRPC_PORT` values in each instance's `.env`. The installer doesn't assign them. - -## Manual Installation - -1. Install the system packages: - - ```bash - sudo apt-get update - sudo apt-get install -y python3 python3-pip python3-venv python3-dev \ - build-essential pkg-config libsqlcipher-dev libmagic1 git make curl - ``` - -2. Install Rust, which builds the payload-specs library: - - ```bash - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source "$HOME/.cargo/env" - ``` - -3. Choose the service user. The installer uses the user running `sudo`; run as root, it creates `relaysms`. To create it yourself: - - ```bash - sudo useradd --system --no-create-home --shell /usr/sbin/nologin relaysms - ``` - -4. Clone the repository with its submodule. The `insteadOf` line fetches the submodule over HTTPS, so no SSH key is needed: - - ```bash - git config --global url."https://github.com/".insteadOf "git@github.com:" - sudo git clone --recurse-submodules \ - https://github.com/smswithoutborders/RelaySMS-Publisher.git \ - /opt/relaysms/relaysms-publisher - cd /opt/relaysms/relaysms-publisher - ``` - -5. Create the virtualenv and build the gRPC code and payload-specs library: - - ```bash - python3 -m venv venv - venv/bin/pip install --upgrade pip - venv/bin/pip install -r requirements.txt - make build - ``` - -6. Create `.env`, then fill it in (see [Configuration](#configuration)): - - ```bash - cp template.env .env - sudo chown root:relaysms .env - sudo chmod 640 .env - ``` - -7. Create the data directories. By default the SQLite database, Celery files and adapters all live under `data/`: - - ```bash - SERVICE_USER=relaysms # or your own user - sudo mkdir -p data/platforms/adapters data/platforms/venvs data/platforms/config data/platforms/state - sudo chown -R "$SERVICE_USER": data && sudo chmod -R 750 data - ``` - - If `.env` moves any of these paths elsewhere, create their parent directories instead: `SQLITE_DATABASE_PATH`, `CELERY_BROKER_DB_PATH`, `CELERY_RESULT_DB_PATH`, `CELERY_BEAT_SCHEDULE_PATH`, `PLATFORMS_ADAPTERS_DIR`, `PLATFORMS_ADAPTERS_VENV_DIR`, `PLATFORMS_ADAPTERS_CONFIG_DIR`, `PLATFORMS_ADAPTERS_STATE_DIR`. - -8. Apply the database migrations: - - ```bash - make migrate - ``` - -9. Install and start the systemd units. They have a `__RW_PATHS__` placeholder for the directories the services may write to: - - ```bash - RW_PATHS="$(pwd)/data" - for unit in deploy/systemd/*; do - sed -e "s/User=relaysms/User=$SERVICE_USER/" \ - -e "s|/opt/relaysms/relaysms-publisher|$(pwd)|g" \ - -e "s|__RW_PATHS__|$RW_PATHS|" \ - "$unit" | sudo tee "/etc/systemd/system/$(basename "$unit")" >/dev/null - done - sudo systemctl daemon-reload - sudo systemctl enable --now relaysms-publisher.target - ``` - -> [!WARNING] -> The services run with `ProtectSystem=strict` and `ProtectHome=true`, so everything outside `ReadWritePaths` is read-only. If `.env` moves a path out of `data/`, add its parent directory to `RW_PATHS`, or writes to it fail. - -## Nginx Reverse Proxy - -`install.sh` can put nginx with a Let's Encrypt certificate in front, using [the template](deploy/nginx/relaysms-publisher-nginx.conf.template). It proxies `/` to `PORT` (REST) and `/publisher.v3.Publisher` to `GRPC_PORT` (gRPC). - -| Flag | Does | -| --- | --- | -| `--site-name DOMAIN` | Sets up nginx for this domain without asking | -| `--letsencrypt-email EMAIL` | Email for Certbot renewal notices (optional) | -| `--skip-nginx` | Skips nginx, even when run interactively | - -```bash -curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ - sudo bash -s -- --site-name publisher.example.com --letsencrypt-email you@example.com -``` - -Re-running it leaves an existing site file alone and skips Certbot when the domain already has a certificate. To set it up by hand: - -```bash -sudo apt-get install -y nginx certbot python3-certbot-nginx -sudo sed -e "s/__SERVER_NAME__/publisher.example.com/g" \ - -e "s/__REST_PORT__/16000/g" -e "s/__GRPC_PORT__/6000/g" \ - deploy/nginx/relaysms-publisher-nginx.conf.template | - sudo tee /etc/nginx/sites-available/publisher.example.com.conf >/dev/null -sudo ln -s /etc/nginx/sites-available/publisher.example.com.conf /etc/nginx/sites-enabled/ -sudo nginx -t && sudo systemctl reload nginx -sudo certbot --nginx -d publisher.example.com --redirect -``` - -## Service Management - -The [README](README.md#manage-the-services) lists the `manage.sh` commands. Run them from the install directory with `sudo`. - -`logs` filters with `-u/--unit` (rest, grpc, worker, beat or smtp; repeatable), `-n/--lines` and `-s/--since`, and `--no-follow` prints and exits: - -```bash -./manage.sh logs --unit rest --unit grpc --since "1 hour ago" --lines 200 -``` - -> [!IMPORTANT] -> Add `--migrate` to `update` when a release has migrations. They run after the dependencies are reinstalled and before the services restart. - -`update` also checks `.env`. Services restart either way, since each fails only on the settings it uses, but the command exits with an error when a setting is invalid. - -`nginx` keeps the previous site file as `.conf.bak` and restores it if the new one fails. Run it after an update that changes the template. - -## The publisher.sh CLI - -`./publisher.sh` runs the admin CLI (`creds`, `platforms`, `gateway-clients`, `config`) from the install directory, with the venv and `.env`, as the service user. - -> [!IMPORTANT] -> Use it instead of `python3 -m publisher`. Run as another user, the CLI can leave adapter files the services can't read. - -```bash -./publisher.sh --help # every command -./publisher.sh env # resolved paths and service user -./publisher.sh shell # a shell as the service user with .env loaded -``` - -[Platform adapters](docs/platforms.md), [gateway clients](docs/gateway-clients.md) and [credentials](README.md#credentials) each have their own guide. Observability is optional; see [observability/README.md](observability/README.md). - -## Configuration - -Settings live in `.env`; `template.env` documents each one. The common ones: - -### Server - -```bash -HOST=127.0.0.1 -PORT=16000 -GRPC_HOST=127.0.0.1 -GRPC_PORT=6000 -GRPC_TLS_ENABLED=false -``` - -### Database - -SQLite is the default: - -```bash -DATABASE_DIALECT=sqlite -SQLITE_DATABASE_PATH=data/relaysms.db -``` - -For MySQL, MariaDB or PostgreSQL, let a script install the server and create a database and user. Both are safe to re-run and write the connection details to `.env`: - -```bash -sudo ./install.sh --setup-db postgres # during install -sudo ./scripts/setup-postgres.sh --db-name relaysms --db-user relaysms # afterwards -sudo ./scripts/setup-mysql.sh --db-name relaysms --db-user relaysms -``` - -`--db-password PASS` sets the password instead of generating one. To use a database server you already have, add `--db-existing` with its details. That only checks the connection and writes `.env`; it never changes the server: - -```bash -sudo ./install.sh --setup-db postgres --db-existing \ - --db-host db.example.com --db-port 5432 \ - --db-name relaysms --db-user relaysms --db-password 'your-password' -``` - -Or set it by hand: - -```bash -DATABASE_DIALECT=mysql # or postgres -MYSQL_HOST=127.0.0.1 # POSTGRES_HOST, POSTGRES_PORT, ... for postgres -MYSQL_PORT=3306 -MYSQL_USER=your_user -MYSQL_PASSWORD=your_password -MYSQL_DATABASE=relaysms_publisher -``` - -### Encryption - -The installer generates these with `openssl rand -hex 32`: - -| Setting | Encrypts | -| --- | --- | -| `DATA_ENCRYPTION_KEY` | Every private key, always | -| `DATABASE_ENCRYPTION_KEY` | The whole database at rest, when `DATABASE_ENCRYPTION_ENABLED=true` (SQLite only) | -| `DATABASE_FIELD_ENCRYPTION_KEY` | Sensitive fields, when `DATABASE_FIELD_ENCRYPTION_ENABLED=true` | - -> [!CAUTION] -> Back the keys up with the database. Data encrypted with a lost or changed key can't be read again. - -### Celery (Worker and Beat) - -The SQLite broker suits light load. For more, use RabbitMQ: - -```bash -sudo ./install.sh --setup-broker rabbitmq # during install -sudo ./scripts/setup-rabbitmq.sh --broker-vhost relaysms --broker-user relaysms # afterwards -``` - -`--broker-password PASS` sets the password. `--broker-existing` (with `--broker-host`) uses a broker you already have: it checks the credentials against the management API (port `15672`, or `--broker-mgmt-port`) and changes nothing. By hand: - -```bash -CELERY_BROKER_TYPE=sqlite # sqlite | redis | rabbitmq -CELERY_BROKER_DB_PATH=data/celery_broker.db -CELERY_RESULT_DB_PATH=data/celery_results.db -# CELERY_REDIS_URL=redis://localhost:6379/0 -# CELERY_RABBITMQ_URL=amqp://user:pass@localhost:5672// -CELERY_BEAT_SCHEDULE_PATH=data/celerybeat-schedule -``` - -### Platform Adapters - -```bash -PLATFORMS_ADAPTERS_DIR=data/platforms/adapters -PLATFORMS_ADAPTERS_VENV_DIR=data/platforms/venvs -PLATFORMS_ADAPTERS_CONFIG_DIR=data/platforms/config # per-adapter credentials.json -PLATFORMS_ADAPTERS_STATE_DIR=data/platforms/state # per-adapter databases and other files -PLATFORMS_GITHUB_ORGS= # orgs administrators may install from over the API -``` - -### Files - -| Path | Holds | -| --- | --- | -| `/opt/relaysms/relaysms-publisher/` | The install | -| `.env` | Configuration (`root:relaysms`, 640) | -| `data/` | SQLite database, Celery files and adapters (service user, 750) | -| `/etc/systemd/system/relaysms-publisher*` | Service units | diff --git a/Makefile b/Makefile index 20c6016..108e1c7 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ CREDS_USER ?= dev SPECS_DIR := lib_relaysms_payload_specs SPECS_LIB := $(SPECS_DIR)/target/release/librelaysms_spec_payload.so -.PHONY: setup build protos specs migrate creds run test coverage test-sdk test-dialects test-e2e docs check clean +.PHONY: setup build protos specs migrate creds run test coverage test-sdk test-dialects test-e2e docs book book-serve check clean ## Set up a development checkout. setup: venv/bin/python .env @@ -22,8 +22,9 @@ venv/bin/python: .env: install -m 600 template.env $@ for key in DATABASE_ENCRYPTION_KEY DATABASE_FIELD_ENCRYPTION_KEY DATA_ENCRYPTION_KEY; do \ - sed -i "s/^$$key=$$/$$key=$$(openssl rand -hex 32)/" $@; \ + sed -i.bak "s/^$$key=$$/$$key=$$(openssl rand -hex 32)/" $@; \ done + rm -f $@.bak ## Generate the gRPC code and the payload-specs bindings. build: protos specs @@ -76,6 +77,14 @@ test-e2e: docs: $(PYTHON) -m publisher.api.rest.openapi +## Build the manual in docs/ into book/. +book: + ./scripts/build-book.sh + +## Serve the manual on http://localhost:3000, rebuilding on changes. +book-serve: + ./scripts/build-book.sh --serve + check: $(PYTHON) -m pre_commit run --all-files diff --git a/README.md b/README.md index e96621c..de2e2fc 100644 --- a/README.md +++ b/README.md @@ -1,146 +1,33 @@ # RelaySMS Publisher -Publishes to online platforms (Gmail, X, Telegram and more) from content sent by SMS, so people can post without an internet connection. +Publishes to online platforms, such as Gmail, from content people send by SMS, so they can post without an internet connection. See [how it works](docs/README.md#how-it-works). -## Contents +## Install -- [How it works](#how-it-works) -- [Requirements](#requirements) -- [Install on a server](#install-on-a-server) -- [Manage the services](#manage-the-services) -- [Run with Docker](#run-with-docker) -- [Development](#development) -- [Configuration](#configuration) -- [Platform adapters](#platform-adapters) -- [Gateway clients](#gateway-clients) -- [Credentials](#credentials) -- [Documentation](#documentation) -- [License](#license) +On Ubuntu 24.04 or Debian 13: -## How it works - -1. A user links an account (Gmail, X, ...) through the gRPC API, which stores its token encrypted. -2. Offline, the user's app encrypts the content and sends it by SMS to a gateway client: a phone number that relays SMS to this server. -3. The payload arrives over REST or SMTP and is queued. -4. A Celery worker decrypts it and publishes it through the platform's adapter. - -## Requirements - -- Ubuntu 24.04 or Debian 13 -- Python 3.12 or newer -- A database: SQLite (the default), MySQL 8.0.28 or newer, MariaDB, or PostgreSQL 12 or newer - -The installer sets up everything else, including the system packages and Rust. - -## Install on a server - -1. Run the installer. It installs to `/opt/relaysms/relaysms-publisher`, generates the encryption keys and starts the services under systemd: - - ```bash - curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | sudo bash - ``` - - With no flags it asks about each option. To choose up front, pass flags after `bash -s --`: - - ```bash - curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ - sudo bash -s -- --site-name publisher.example.com --setup-db postgres - ``` - - | Flag | Does | - | --- | --- | - | `--site-name DOMAIN` | Puts nginx with a Let's Encrypt certificate in front; `--skip-nginx` skips it | - | `--setup-db mysql\|postgres` | Installs the database server and writes its details to `.env` ([details](INSTALL.md#database)) | - | `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker, for heavier load ([details](INSTALL.md#celery-worker-and-beat)) | - | `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](observability/README.md)) | - | `--install-dir PATH`, `--instance-name NAME` | Installs elsewhere, or a second copy on the same host ([details](INSTALL.md#running-multiple-instances)) | - - `install.sh --help` lists every flag. - -2. Go to the install directory and check the configuration: - - ```bash - cd /opt/relaysms/relaysms-publisher - sudo ./manage.sh check - ``` - -3. Create an administrator credential for the REST API: - - ```bash - ./publisher.sh creds create --username ops --administrator - ``` - - > [!IMPORTANT] - > The password is printed once. Store it before closing the terminal. - -4. Add the platform adapters you want to offer, then follow each adapter's README to configure it: - - ```bash - ./publisher.sh platforms add https://github.com/smswithoutborders/gmail-oauth2-adapter - ``` - -5. Register the gateway clients that relay SMS to this server: - - ```bash - ./publisher.sh gateway-clients create --msisdn +237123456789 --protocols https - ``` - -6. Check that the services are up: - - ```bash - sudo ./manage.sh status - curl http://127.0.0.1:16000/health - ``` - -[INSTALL.md](INSTALL.md) covers installing by hand and every setting. - -## Manage the services - -From the install directory: - -| Command | Does | -| --- | --- | -| `sudo ./manage.sh status` | Show each service's state | -| `sudo ./manage.sh logs` | Follow the service logs | -| `sudo ./manage.sh start`, `stop`, `restart` | Control every service together | -| `sudo ./manage.sh enable`, `disable` | Start the services on boot, or stop doing so | -| `sudo ./manage.sh update` | Pull the latest code, rebuild and restart; add `--migrate` when the release has migrations | -| `sudo ./manage.sh migrate` | Apply database migrations | -| `sudo ./manage.sh check` | Report every missing or invalid setting | -| `sudo ./manage.sh nginx [DOMAIN]` | Re-render the nginx site and reattach or obtain its certificate | -| `sudo ./manage.sh uninstall` | Remove the services and the install directory | - -See [Service Management](INSTALL.md#service-management) for log filters and what `update` and `nginx` do. - -## Run with Docker - -1. Clone the repository with its submodule: - - ```bash - git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git - cd RelaySMS-Publisher - ``` - -2. Create `.env` with fresh encryption keys: +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash +``` - ```bash - make .env - ``` +[Install on a server](docs/installation/server.md) covers the installer's options and the steps after it. To try it on your machine with Docker first, follow the [quick start](docs/getting-started/quick-start.md). -3. Build and start: +## Documentation - ```bash - docker compose up -d --build - ``` +The [manual](docs/README.md) starts with: -The container applies pending migrations, then starts every service. It listens on `PORT` (16000) and `GRPC_PORT` (6000) from `.env`, and keeps its data in `./data`. +- [Quick start](docs/getting-started/quick-start.md) +- [Settings reference](docs/configuration/settings.md) +- [Managing adapters](docs/platforms/manage.md) and the [provider guides](docs/platforms/manage.md#provider-guides) +- [Connecting a relay](docs/gateway-clients/connect-a-relay.md) +- [Troubleshooting](docs/operating/troubleshooting.md) -> [!CAUTION] -> Back up `.env` with `./data`: its [encryption keys](INSTALL.md#encryption) are the only way to read the data. +`sudo ./manage.sh docs ` serves the manual from your own server. -## Development +## Contributing -[CONTRIBUTING.md](CONTRIBUTING.md) walks through setting up, running the services locally, testing and making common changes. In short: +[CONTRIBUTING.md](CONTRIBUTING.md) covers setup, running the services locally, testing and the workflow. To start: ```bash git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git @@ -148,90 +35,12 @@ cd RelaySMS-Publisher make setup ``` -Then follow [Run locally](CONTRIBUTING.md#run-locally). To fill a local database with sample data: - -```bash -source venv/bin/activate -python -m publisher seed stats --count 5000 --days 90 # publication stats -python -m publisher seed creds --count 5 # credentials; prints their passwords -python -m publisher seed platforms # adapter rows, without their files -python -m publisher seed gateway-clients --count 10 # gateway clients, some disabled -``` - -## Configuration - -Settings live in `.env`. `template.env` lists every setting with its default and what it does, and `python -m publisher config check` (or `sudo ./manage.sh check` on a server) reports any that are missing or invalid. - -| Settings | Documented in | -| --- | --- | -| Server, database, encryption, Celery, adapters | [INSTALL.md](INSTALL.md#configuration) | -| REST API login and sessions (`AUTH_*`) | [REST API](docs/rest.md#authentication) | -| SMTP transport (`SMTP_*`) | [SMTP Transport](docs/smtp.md#configuration) | -| Tracing, metrics and logs (`OTEL_*`) | [Observability](observability/README.md) | -| Offline publishing (`OFFLINE_PUBLISH_*`) | [Below](#offline-publishing) | - -### Offline Publishing - -Offline payloads are tagged with the protocol they arrived on: `https` for [REST `/publications`](docs/rest.md#publishing) and `smtp` for the [SMTP transport](docs/smtp.md). - -```bash -OFFLINE_PUBLISH_ALLOWED_PROTOCOLS=smtp # comma-separated; empty allows every protocol -OFFLINE_PUBLISH_SHARED_SECRET= # 64-char hex; required when https is allowed -``` +Adapters are built with the [adapter SDK](sdk/README.md). -- Offline payloads from a protocol not in `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` are discarded. -- Offline payloads over `https` must carry `OFFLINE_PUBLISH_SHARED_SECRET` in the request's `tag` field. -- Allowing `https`, or every protocol, without the secret fails `config check` and startup. - -> [!TIP] -> For a Twilio number, run [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) with `FORWARD_URLS_JSON` set to this server's `/v1/publications` URL and `FORWARD_TAG` set to `OFFLINE_PUBLISH_SHARED_SECRET`, then allow `https`. - -## Platform adapters - -Each platform is served by an adapter installed from its own repository. `GET /v1/platforms` lists the enabled ones. - -- [Gmail](https://github.com/smswithoutborders/gmail-oauth2-adapter) -- [X (formerly Twitter)](https://github.com/smswithoutborders/twitter-oauth2-adapter) -- [Telegram](https://github.com/smswithoutborders/telegram-pnba-adapter) -- [Slack](https://github.com/smswithoutborders/slack-oauth2-adapter) -- [Bluesky](https://github.com/smswithoutborders/bluesky-oauth2-adapter) -- [Mastodon](https://github.com/smswithoutborders/mastodon-oauth2-adapter) - -[docs/platforms.md](docs/platforms.md) covers adding, updating, disabling and removing adapters. Each adapter's README covers its own settings. - -## Gateway clients - -Gateway clients are the phone numbers that relay SMS to this server. `GET /v1/gateway-clients` lists the enabled ones, and [docs/gateway-clients.md](docs/gateway-clients.md) covers managing them. - -## Credentials - -Credentials log in to the REST API, each with its own scopes. One holding every scope is an administrator. They can also be managed over the [REST API](docs/rest.md#managing-credentials). - -```bash -./publisher.sh creds scopes # list scopes -./publisher.sh creds create --username ops --administrator # prints the password once -./publisher.sh creds create --username analyst --scope stats:publications:read -./publisher.sh creds set-scopes --username analyst --scope stats:publications:read --scope stats:publications:reasons -./publisher.sh creds list -./publisher.sh creds reset-password --username analyst -./publisher.sh creds disable --username analyst -./publisher.sh creds enable --username analyst -./publisher.sh creds revoke-sessions --username analyst -./publisher.sh creds delete --username analyst -``` - -## Documentation +## Security -- [Installation Guide](INSTALL.md): installing by hand and every setting -- [REST API](docs/rest.md): publishing, authentication and the [OpenAPI reference](docs/openapi.json) -- [gRPC API](docs/grpc.md): linking accounts and syncing keys -- [SMTP Transport](docs/smtp.md): publishing payloads sent by email -- [Platform Adapters](docs/platforms.md): installing and managing adapters -- [Gateway Clients](docs/gateway-clients.md): managing gateway clients -- [Observability](observability/README.md): tracing, metrics, logs and uptime monitoring -- [Reference Client](tools/README.md): exercising the gRPC and REST flows by hand -- [Contributing](CONTRIBUTING.md): setup, workflow, conventions and testing +Report vulnerabilities privately, as [SECURITY.md](SECURITY.md) describes. ## License -Licensed under the GNU General Public License v3. See [LICENSE](LICENSE.md). +Licensed under the GNU General Public License v3.0. See [LICENSE.md](LICENSE.md). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..6334e0e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Security policy + +Don't open a public issue for a vulnerability. Email **** with: + +- a description of the issue and its impact; +- the affected versions or commits; +- steps to reproduce it. + +This covers the Publisher, its adapter SDK in `sdk/` and the platform adapters in the [smswithoutborders](https://github.com/smswithoutborders) organization. diff --git a/book.toml b/book.toml new file mode 100644 index 0000000..a3e9c80 --- /dev/null +++ b/book.toml @@ -0,0 +1,20 @@ +[book] +title = "RelaySMS Publisher" +description = "Install, configure and run the RelaySMS Publisher." +language = "en" +src = "docs" + +[build] +create-missing = false +# Without the index preprocessor, README.md pages stay README.html, so links to them work in the book as on GitHub. +use-default-preprocessors = false + +[preprocessor.mermaid] +command = "mdbook-mermaid" + +[output.html] +git-repository-url = "https://github.com/smswithoutborders/RelaySMS-Publisher" +edit-url-template = "https://github.com/smswithoutborders/RelaySMS-Publisher/edit/staging/{path}" +additional-js = ["mermaid.min.js", "mermaid-init.js"] +additional-css = ["docs/theme/custom.css"] +no-section-label = true diff --git a/deploy/nginx/relaysms-publisher-docs-nginx.conf.template b/deploy/nginx/relaysms-publisher-docs-nginx.conf.template new file mode 100644 index 0000000..bb93a12 --- /dev/null +++ b/deploy/nginx/relaysms-publisher-docs-nginx.conf.template @@ -0,0 +1,30 @@ +# RelaySMS Publisher manual - nginx static site template +# +# Installed to /etc/nginx/sites-available/.conf by install.sh +# --docs-site-name. Only the port 80 block is defined; certbot adds HTTPS. + +server { + listen 80; + listen [::]:80; + server_name __SERVER_NAME__; + + server_tokens off; + root __BOOK_DIR__; + + access_log /var/log/nginx/__SERVER_NAME__.access.log combined buffer=512k flush=1m; + error_log /var/log/nginx/__SERVER_NAME__.error.log warn; + + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "DENY" always; + add_header Referrer-Policy "no-referrer" always; + add_header Cross-Origin-Opener-Policy "same-origin" always; + # mdBook pages and Mermaid diagrams use inline scripts and styles. + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'none'" always; + + gzip on; + gzip_vary on; + gzip_types text/css application/javascript application/json image/svg+xml; + + error_page 404 /404.html; +} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..39c4401 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,76 @@ +# RelaySMS Publisher + +Publishes to online platforms, such as Gmail, from content people send by SMS, so they can post without an internet connection. + +## How it works + +```mermaid +--- +config: + flowchart: + curve: linear +--- +flowchart TB + app["RelaySMS app"] + gateway["Gateway client"] + mailbox["Mailbox"] + platform["Online platforms"] + + subgraph publisher["Publisher"] + grpc["gRPC API"] + rest["REST API"] + smtp["Email listener"] + db[("Database")] + worker["Celery worker"] + adapter["Platform adapter"] + %% Keeps the title clear of the arrows entering the box. + spacer[" "]:::hidden + spacer ~~~ worker + end + + app -- "link an account, online" --> grpc + app -- "encrypted SMS" --> gateway + gateway -- "HTTPS" --> rest + gateway -- "email" --> mailbox + mailbox -- "IMAP" --> smtp + grpc -- "token, keys" --> db + db --> worker + rest --> worker + smtp --> worker + grpc --> adapter + worker --> adapter + adapter --> platform + + classDef hidden fill:none,stroke:none +``` + +1. While online, a user links an account through the gRPC API. The Publisher stores its token encrypted. +2. Offline, the app encrypts the content and sends it by SMS to a gateway client, a phone number that relays SMS. +3. The gateway client posts the payload to the REST API, or emails it to a mailbox the Publisher reads. +4. A Celery worker decrypts the payload and publishes it through the platform's adapter. + +## Key terms + +- **Gateway client**: a phone number that relays SMS to the Publisher, such as an Android phone running Deku SMS. See [Connecting a relay](gateway-clients/connect-a-relay.md). +- **Adapter** (platform adapter): the plugin that publishes to one platform, such as Gmail, installed from its own repository. See [Managing adapters](platforms/manage.md). +- **Linked account**: a user's account on a platform, authorized once while online. The Publisher stores its token encrypted. +- **OAuth2 and PNBA**: how accounts link. OAuth2 goes through the platform's consent page, as for Gmail. Phone number-based authentication (PNBA) uses a code sent to the user's phone, as for Telegram. The [provider guides](platforms/manage.md#provider-guides) give each platform's protocol. +- **Offline payload**: content sent without a linked account, published through [RelaySMS Mail](platforms/relaysms-mail.md). See [Offline publishing](configuration/offline-publishing.md). + +The [glossary](reference/glossary.md) defines the rest, and [Architecture](explanation/architecture.md) shows every component. + +## Where to start + +| To | Read | +| --- | --- | +| Try the Publisher on your machine | [Quick start](getting-started/quick-start.md) | +| Run it in production | [Requirements](installation/requirements.md), then [Install on a server](installation/server.md) | +| Offer a platform | [Managing adapters](platforms/manage.md) | +| Relay SMS to the Publisher | [Managing gateway clients](gateway-clients/manage.md) | +| Build a client | [gRPC API](reference/grpc-api.md) and [REST API](reference/rest-api.md) | +| Understand the protocol | [Security model](explanation/security-model.md) and the [RelaySMS whitepaper](explanation/relaysms-whitepaper.pdf) | + +## Getting help + +- Ask questions and report bugs in the [issue tracker](https://github.com/smswithoutborders/RelaySMS-Publisher/issues); [CONTRIBUTING.md](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/CONTRIBUTING.md#ways-to-contribute) says what a useful report has. +- Report vulnerabilities privately, as [SECURITY.md](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/SECURITY.md) describes. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md new file mode 100644 index 0000000..b67ba52 --- /dev/null +++ b/docs/SUMMARY.md @@ -0,0 +1,68 @@ +# Summary + +[Introduction](README.md) + +# Getting started + +- [Quick start](getting-started/quick-start.md) + +# Installation + +- [Requirements](installation/requirements.md) +- [Install on a server](installation/server.md) +- [Install with Docker](installation/docker.md) +- [Install by hand](installation/manual.md) +- [Database](installation/database.md) +- [Celery broker](installation/celery-broker.md) +- [Reverse proxy and TLS](installation/reverse-proxy.md) +- [Multiple instances](installation/multiple-instances.md) + +# Configuration + +- [Settings reference](configuration/settings.md) +- [Encryption keys](configuration/encryption-keys.md) +- [Offline publishing](configuration/offline-publishing.md) +- [SMTP transport](configuration/smtp-transport.md) + +# Platform adapters + +- [Managing adapters](platforms/manage.md) +- [Gmail](platforms/gmail.md) +- [Mastodon](platforms/mastodon.md) +- [Bluesky](platforms/bluesky.md) +- [Telegram](platforms/telegram.md) +- [RelaySMS Mail](platforms/relaysms-mail.md) + +# Gateway clients + +- [Managing gateway clients](gateway-clients/manage.md) +- [Connecting a relay](gateway-clients/connect-a-relay.md) + +# Operating + +- [Services](operating/services.md) +- [Credentials and access](operating/credentials.md) +- [Upgrading](operating/upgrading.md) +- [Backup and restore](operating/backup-restore.md) +- [Observability](operating/observability.md) +- [Audit log](operating/audit-log.md) +- [Security hardening](operating/security-hardening.md) +- [Troubleshooting](operating/troubleshooting.md) + +# Reference + +- [Command-line tools](reference/cli.md) +- [REST API](reference/rest-api.md) +- [gRPC API](reference/grpc-api.md) +- [Glossary](reference/glossary.md) + +# Explanation + +- [Architecture](explanation/architecture.md) +- [Security model](explanation/security-model.md) +- [Data and retention](explanation/data-retention.md) + +# Developers + +- [Writing an adapter](developers/writing-an-adapter.md) +- [Reference client](developers/reference-client.md) diff --git a/docs/configuration/encryption-keys.md b/docs/configuration/encryption-keys.md new file mode 100644 index 0000000..cf99b82 --- /dev/null +++ b/docs/configuration/encryption-keys.md @@ -0,0 +1,12 @@ +# Encryption keys + +Three keys in `.env` protect the data. The installer and `make .env` generate them. Each is 64 hex characters; `openssl rand -hex 32` makes one, and `manage.sh check` reports a missing or malformed key. + +| Setting | Protects | Used | +| --- | --- | --- | +| `DATA_ENCRYPTION_KEY` | The server's private keys, and the keyed hash that replaces each sender's address | Always | +| `DATABASE_FIELD_ENCRYPTION_KEY` | Linked accounts' platform tokens | When field encryption is on | +| `DATABASE_ENCRYPTION_KEY` | The whole SQLite database file (SQLCipher) | When encryption is on and the database is SQLite | + +> [!CAUTION] +> Back up `.env` whenever you back up the database, store it apart ([Backup and restore](../operating/backup-restore.md)), and never change a key once the install holds data. Data encrypted with a lost or changed key can't be read again, and a new `DATA_ENCRYPTION_KEY` also gives every sender a new hash. diff --git a/docs/configuration/offline-publishing.md b/docs/configuration/offline-publishing.md new file mode 100644 index 0000000..cf23460 --- /dev/null +++ b/docs/configuration/offline-publishing.md @@ -0,0 +1,33 @@ +# Offline publishing + +Offline payloads let people send email through [RelaySMS Mail](../platforms/relaysms-mail.md) without linking an account; linked accounts need none of this ([settings](settings.md#offline-publishing)). The app encrypts an offline payload to the server's identity key, and the Publisher publishes it through the `rmail` adapter, which must be installed. Anyone who can reach an allowed protocol can send one. + +Each payload is tagged with the protocol it arrived on: + +- `https`: [REST `POST /v1/publications`](../reference/rest-api.md#publishing) +- `smtp`: the [SMTP transport](smtp-transport.md) + +- An offline payload from a protocol not in `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` is discarded. +- An offline payload over `https` must carry `OFFLINE_PUBLISH_SHARED_SECRET` in the request's `tag` field, or it's discarded. +- Leaving `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` empty, or listing `https`, without the secret fails `manage.sh check`, and the services refuse to start. +- A discarded payload is logged and recorded as `failed` in the publication stats, with reason `protocol_not_allowed`, `offline_tag_missing` or `offline_tag_invalid`. `POST /v1/publications` still returns success, since the check runs in the queue. + +## Allowing https + +1. Generate a secret: + + ```bash + openssl rand -hex 32 + ``` + +2. Set `OFFLINE_PUBLISH_SHARED_SECRET` to it, and add `https` to `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS`, such as `smtp,https`. +3. Check and restart: + + ```bash + sudo ./manage.sh check + sudo ./manage.sh restart + ``` + +## Next steps + +[Connect each relay](../gateway-clients/connect-a-relay.md) that posts over `https` with the secret as its `tag` (`FORWARD_TAG` for Twilio). diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md new file mode 100644 index 0000000..30a1693 --- /dev/null +++ b/docs/configuration/settings.md @@ -0,0 +1,141 @@ +# Settings reference + +Settings live in `.env` in the install directory. `template.env` lists each one with its default. + +- `sudo ./manage.sh check` reports missing or invalid settings; each service refuses to start on an error in the settings it uses. +- Restart the services after a change: `sudo ./manage.sh restart`. +- After an update, `sudo ./sync-env.sh` adds the settings from `template.env` that `.env` is missing, and shows the change before writing it. +- On Docker, check with `docker compose exec publisher python -m publisher config check` and apply a change with `docker compose up -d`. + +## Server + +| Setting | Default | Description | +| --- | --- | --- | +| `HOST` | `127.0.0.1` | REST API address. Docker always binds `0.0.0.0`. | +| `PORT` | `16000` | REST API port | +| `WORKERS` | None | REST API worker processes: `1` by default under `scripts/run.sh`, `4` on Docker and server installs | +| `GRPC_HOST` | `127.0.0.1` | gRPC address. Docker always binds `0.0.0.0`. | +| `GRPC_PORT` | `6000` | gRPC port | +| `GRPC_MAX_WORKERS` | `10` | gRPC worker threads | +| `GRPC_TLS_ENABLED` | `false` | Serves TLS on `GRPC_PORT` itself; leave off behind nginx ([details](../installation/reverse-proxy.md#grpc-tls-without-nginx)) | +| `GRPC_TLS_CERT_FILE`, `GRPC_TLS_KEY_FILE` | None | Certificate and key. Required when `GRPC_TLS_ENABLED=true`. | +| `NONCE_TTL_SECONDS` | `600` | How far a gRPC request's timestamp may be from the server's clock; nonces are remembered this long to refuse replays | + +## Database + +See [Database](../installation/database.md) and [Encryption keys](encryption-keys.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `DATABASE_DIALECT` | `sqlite` | `sqlite`, `mysql` (MySQL and MariaDB) or `postgres` | +| `SQLITE_DATABASE_PATH` | `data/relaysms.db` | SQLite database file | +| `MYSQL_HOST` | `127.0.0.1` | MySQL server | +| `MYSQL_PORT` | `3306` | MySQL port | +| `MYSQL_USER`, `MYSQL_PASSWORD` | None | Required for `mysql` | +| `MYSQL_DATABASE` | `relaysms` | Required for `mysql`; created if it doesn't exist | +| `POSTGRES_HOST` | `127.0.0.1` | PostgreSQL server | +| `POSTGRES_PORT` | `5432` | PostgreSQL port | +| `POSTGRES_USER`, `POSTGRES_PASSWORD` | None | Required for `postgres` | +| `POSTGRES_DATABASE` | `relaysms` | Required for `postgres`; created if it doesn't exist | +| `DATABASE_ENCRYPTION_ENABLED` | `true` (`false` if unset) | Encrypts the SQLite file with SQLCipher. No effect on MySQL or PostgreSQL. | +| `DATABASE_ENCRYPTION_KEY` | None | Required for SQLite when encryption is on | +| `DATABASE_FIELD_ENCRYPTION_ENABLED` | `true` (`false` if unset) | Encrypts linked accounts' platform tokens | +| `DATABASE_FIELD_ENCRYPTION_KEY` | None | Required when field encryption is on | +| `DATA_ENCRYPTION_KEY` | None | Required. Encrypts the server's private keys, and keys the hash that replaces each sender's address. | + +## Celery + +See [Celery broker](../installation/celery-broker.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `CELERY_BROKER_TYPE` | `sqlite` | `sqlite`, `redis` or `rabbitmq` | +| `CELERY_WORKER_CONCURRENCY` | `1` for `sqlite`, else `4` | Tasks the worker runs at once | +| `CELERY_BROKER_DB_PATH` | `data/celery_broker.db` | Broker file when `CELERY_BROKER_TYPE=sqlite` | +| `CELERY_RESULT_DB_PATH` | `data/celery_results.db` | Result file when `CELERY_BROKER_TYPE=sqlite` | +| `CELERY_REDIS_URL` | `redis://localhost:6379/0` | Broker and result backend when `CELERY_BROKER_TYPE=redis` | +| `CELERY_RABBITMQ_URL` | `amqp://guest:guest@localhost:5672//` | Broker when `CELERY_BROKER_TYPE=rabbitmq` | +| `CELERY_BEAT_SCHEDULE_PATH` | `data/celerybeat-schedule` | Beat's schedule file | +| `CELERY_CLEANUP_CRON` | `0 */3 * * *` | When to clean up expired data ([below](#cleanup-jobs)) | +| `CELERY_TOKEN_CLEANUP_CRON` | `0 3 * * *` | When to remove idle tokens ([below](#cleanup-jobs)) | + +### Cleanup jobs + +Both schedules are 5-field cron expressions in UTC. + +- `CELERY_CLEANUP_CRON` deletes incomplete payloads, ended web sessions, and audit events and adapter jobs past retention, and fails abandoned adapter jobs. +- `CELERY_TOKEN_CLEANUP_CRON` deletes tokens idle past `TOKEN_IDLE_MAX_AGE_DAYS` and revokes them on their platform. + +## Retention + +| Setting | Default | Description | +| --- | --- | --- | +| `PAYLOAD_SESSION_MAX_AGE_HOURS` | `3` | Hours before an incomplete multi-part payload is discarded | +| `TOKEN_IDLE_MAX_AGE_DAYS` | `90` | Days without use before a linked account's token is deleted and revoked on its platform | +| `AUDIT_RETENTION_DAYS` | `365` | Days to keep audit events and adapter install and update jobs | + +## Offline publishing + +See [Offline publishing](offline-publishing.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` | `smtp` (empty if unset) | Comma-separated protocols whose offline payloads publish: `https`, `smtp`. Empty allows both. | +| `OFFLINE_PUBLISH_SHARED_SECRET` | None | 64 hex characters. Required when `https` is allowed. | + +## SMTP transport + +See [SMTP transport](smtp-transport.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `SMTP_TRANSPORT_ENABLED` | `false` | Runs the email listener | +| `SMTP_IMAP_SERVER`, `SMTP_IMAP_USERNAME`, `SMTP_IMAP_PASSWORD` | None | IMAP host and login. Required when the transport is enabled. | +| `SMTP_IMAP_PORT` | `993` | IMAP port | +| `SMTP_IMAP_MAIL_FOLDER` | `INBOX` | Comma-separated folders to watch, such as `INBOX,Spam` | +| `SMTP_TLS_CLIENT_CERTIFICATE`, `SMTP_TLS_CLIENT_KEY` | None | mTLS client certificate and key, if the provider requires one. Set both or neither. | +| `SMTP_ALLOWED_SENDERS` | None | Comma-separated addresses and domains allowed to send. Empty allows nothing. | +| `SMTP_TRUSTED_AUTHSERV_ID` | None | `authserv-id` whose `Authentication-Results` verdicts are trusted, such as `mx.google.com`. Required when the transport is enabled. | +| `SMTP_REQUIRE_DKIM`, `SMTP_REQUIRE_SPF` | `true` | Require `dkim=pass` and `spf=pass` | +| `SMTP_VERIFY_DKIM_INDEPENDENTLY` | `false` | Also re-verifies the DKIM signature against DNS | + +## Platform adapters + +See [Managing adapters](../platforms/manage.md#files). Relative paths are from the install directory. + +| Setting | Default | Description | +| --- | --- | --- | +| `PLATFORMS_ADAPTERS_DIR` | `data/platforms/adapters` | Directory for adapters' code | +| `PLATFORMS_ADAPTERS_VENV_DIR` | `data/platforms/venvs` | Directory for adapters' virtualenvs | +| `PLATFORMS_ADAPTERS_CONFIG_DIR` | `data/platforms/config` | Directory for adapters' `credentials.json` files | +| `PLATFORMS_ADAPTERS_STATE_DIR` | `data/platforms/state` | Directory for files adapters write, such as databases | +| `PLATFORMS_GITHUB_ORGS` | None | Comma-separated GitHub orgs that administrators may install and update adapters from over the REST API. Empty refuses REST installs; `publisher.sh platforms` is unaffected. | + +## REST API access + +See [Credentials and access](../operating/credentials.md#logging-in). + +| Setting | Default | Description | +| --- | --- | --- | +| `AUTH_WEB_ORIGINS` | None | Comma-separated web client origins allowed to use a session cookie, such as `https://console.example.com` | +| `AUTH_SESSION_COOKIE_SECURE` | `true` | Sends the session cookie over HTTPS only. Set `false` only for local HTTP development. | +| `AUTH_SESSION_IDLE_MINUTES` | `30` | Minutes of inactivity that end a web session | +| `AUTH_SESSION_MAX_HOURS` | `12` | Hours after login when a web session ends, even if it's active | +| `API_DOCS_ENABLED` | `false` | Serves interactive API docs at `/docs` and the schema at `/openapi.json`. For development. | + +`AUTH_WEB_ORIGINS` origins must be on the same site as the API. Leave it empty when the web client is served from the API's own origin. + +## Logging and monitoring + +See [Observability](../operating/observability.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warning`, `error` or `critical` | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | None | OTLP collector URL for traces, metrics and logs, such as `http://localhost:4317`. Needs `requirements-observability.txt`. Blank turns all three off. | +| `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | `grpc` or `http/protobuf` | +| `OTEL_TRACES_SAMPLER` | `parentbased_traceidratio` | Which traces to keep | +| `OTEL_TRACES_SAMPLER_ARG` | `1.0` | Fraction of traces kept, `0` to `1` | +| `OTEL_LOGS_EXPORTER` | `otlp` | `otlp` sends logs to the collector; `none` doesn't | +| `OTEL_PYTHON_LOG_CORRELATION` | `true` | Adds trace and span IDs to log records | +| `UPTIME_KUMA_WORKER_PUSH_URL`, `UPTIME_KUMA_SMTP_PUSH_URL` | None | Uptime Kuma push URLs for the worker and email listener heartbeats. Blank turns them off. | diff --git a/docs/configuration/smtp-transport.md b/docs/configuration/smtp-transport.md new file mode 100644 index 0000000..4bbf4b2 --- /dev/null +++ b/docs/configuration/smtp-transport.md @@ -0,0 +1,67 @@ +# SMTP transport + +Use this when gateway clients relay payloads by email instead of HTTPS. The listener watches a mailbox over IMAP, checks each sender, and queues the payload for publication. + +## Set it up + +1. Create a mailbox that the gateway clients email, with IMAP access. +2. In `.env`, set `SMTP_IMAP_SERVER`, `SMTP_IMAP_USERNAME` and `SMTP_IMAP_PASSWORD`. To also watch a spam folder, set `SMTP_IMAP_MAIL_FOLDER`, such as `INBOX,Spam`. +3. Set `SMTP_ALLOWED_SENDERS` to the addresses the gateway clients send from ([allowlist](#allowlist)). +4. Set `SMTP_TRUSTED_AUTHSERV_ID` to the `authserv-id` the mailbox's server writes: the first word of the `Authentication-Results` header in a received email, such as `mx.google.com` ([authentication](#authentication)). +5. Set `SMTP_TRANSPORT_ENABLED=true`, then check and restart: + + ```bash + sudo ./manage.sh check + sudo ./manage.sh restart + ``` + +[Settings reference](settings.md#smtp-transport) lists every `SMTP_*` setting. + +## Check it + +```bash +sudo ./manage.sh logs --unit smtp --no-follow --lines 20 +``` + +You should see `Connected to mailbox `. + +## How it works + +1. Waits for unseen mail in `SMTP_IMAP_MAIL_FOLDER` with IMAP IDLE. +2. Rejects each email unless the sender is allowlisted and authenticated. +3. Parses the body as JSON and validates the payload. +4. Queues the payload for publication, tagged with protocol `smtp`. Whether offline payloads from email publish depends on [offline publishing](offline-publishing.md). +5. Deletes each handled email. An email that hits an unexpected error stays in the folder and is retried on the next cycle. + +### Allowlist + +`SMTP_ALLOWED_SENDERS` lists exact addresses and domains, comma-separated. Empty allows nothing. + +> [!WARNING] +> A domain entry trusts every authenticated sender on that domain, since DKIM vouches for the domain, not the mailbox. List an exact address when only one sender should be trusted. + +### Authentication + +The listener can't re-check SPF, which needs the sender's IP. It reads the `Authentication-Results` header ([RFC 8601](https://www.rfc-editor.org/rfc/rfc8601)) that the mailbox's own server added, and only from the `authserv-id` in `SMTP_TRUSTED_AUTHSERV_ID`, since a sender can forge such a header from anyone else. + +## Message format + +The body of each email must be a JSON object: + +```json +{ + "address": "5447c1f50558292bd9df723f9fdc0b06b892199c7dcfaad5164f2d94dfd3470a", + "dialing_code": "237", + "text": "" +} +``` + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `address` | string | Yes | Sender ID from the gateway client, or an E.164 phone number | +| `dialing_code` | string | No | Sender's country calling code, for publication stats | +| `text` | string | Yes | Base64-encoded serialized payload | + +## Next steps + +To publish offline payloads that arrive by email, allow `smtp` in [Offline publishing](offline-publishing.md). diff --git a/tools/README.md b/docs/developers/reference-client.md similarity index 68% rename from tools/README.md rename to docs/developers/reference-client.md index cb7d1b6..cb44553 100644 --- a/tools/README.md +++ b/docs/developers/reference-client.md @@ -1,24 +1,24 @@ -# Reference Client +# Reference client -`tools/client.py` drives the gRPC and REST flows by hand, the way a phone app would. It keeps tokens and key pools in local state between runs. +`tools/client.py` drives the gRPC and REST flows by hand, the way a phone app would. It keeps tokens and key pools in `tools/db.json` between runs. ```bash pip install -r requirements.txt python -m tools.client [ARGS] ``` -## Connection Options +## Connection options | Option | Default | Does | | --- | --- | --- | | `--host`, `--port` | `127.0.0.1`, `6000` | gRPC server | | `--tls` | off | Use TLS for gRPC | | `--rest-api` | `http://localhost:16000` | REST API base URL | -| `--platform`, `-p` | | Platform name, such as `gmail` or `telegram` | -| `--phone-number` | | Phone number, for PNBA | -| `--request-identifier` | | Optional request identifier | +| `--platform`, `-p` | None | Platform name, such as `gmail` | +| `--phone-number` | None | Phone number, for phone number-based authentication (PNBA), such as for Telegram | +| `--request-identifier` | None | Optional request identifier | -## Linking Accounts +## Linking accounts | Command | Does | | --- | --- | @@ -33,25 +33,24 @@ python -m tools.client [ARGS] ```bash python -m tools.client get-oauth2-url --platform gmail python -m tools.client exchange-oauth2-code --platform gmail --code -python -m tools.client get-pnba-code --platform telegram --phone-number +237123456789 -python -m tools.client exchange-pnba-code --platform telegram --phone-number +237123456789 --code +python -m tools.client get-pnba-code --platform telegram --phone-number +237600000000 +python -m tools.client exchange-pnba-code --platform telegram --phone-number +237600000000 --code ``` -> [!NOTE] -> With several stored tokens, `sync-keys` and the revoke commands ask which one to use. Pass `--token` to choose up front. +With several stored tokens, `sync-keys` and the revoke commands ask which one to use. Pass `--token` to choose up front. ## Publishing -`send` encrypts a message and posts it to `POST /v1/publications` with `--address` as the sender. Without an attachment it's one request; with one, the payload is split into SMS-sized segments sent `--interval` seconds apart. +`send` encrypts a message and posts it to `POST /v1/publications` with `--address` as the sender. With an attachment, the payload is split into SMS-sized segments sent `--interval` seconds apart. ```bash -python -m tools.client send --platform gmail --address +237123456789 \ +python -m tools.client send --platform gmail --address +237600000000 \ --to friend@example.com --subject "Hello" --body "Test message" -python -m tools.client send --platform gmail --address +237123456789 \ +python -m tools.client send --platform gmail --address +237600000000 \ --to friend@example.com --subject "Hello" --body "See attached" --attachment ./file.pdf --interval 2.5 --shuffle -python -m tools.client send --offline --platform rmail --address +237123456789 \ +python -m tools.client send --offline --platform rmail --address +237600000000 \ --to friend@example.com --subject "Hello" --body "No token needed" --tag ``` @@ -69,8 +68,6 @@ python -m tools.client send --offline --platform rmail --address +237123456789 \ | `--offline` | Uses offline-first encryption, which needs no linked token (`rmail` only) | | `--tag` | The server's `OFFLINE_PUBLISH_SHARED_SECRET`, when it sets one | -> [!NOTE] -> Each online send uses up one ephemeral key pair. Run `sync-keys` when the pool runs low. Offline sends use none. +Each online send uses up one ephemeral key pair. Run `sync-keys` when the pool runs low. Offline sends use none. -> [!TIP] -> `--shuffle --dry-run` shows the segment order before a live send, to check the server reassembles out-of-order segments. +`--shuffle --dry-run` shows the segment order before a live send, to check the server reassembles out-of-order segments. diff --git a/docs/developers/writing-an-adapter.md b/docs/developers/writing-an-adapter.md new file mode 100644 index 0000000..c0d44f4 --- /dev/null +++ b/docs/developers/writing-an-adapter.md @@ -0,0 +1,30 @@ +# Writing an adapter + +An adapter publishes to one platform. It's a Python package in its own repository, built with the [RelaySMS Adapter SDK](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk), which the Publisher runs in its own virtualenv and calls over JSON-RPC. + +## Requirements + +- Python 3.12 or newer +- A GitHub repository +- A developer account and API credentials for the platform + +## Steps + +1. Copy the SDK's [echo adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk/examples/echo-adapter) into a new repository named after the platform and protocol, such as `example-oauth2-adapter`. +2. Subclass the SDK's base class for its protocol, such as `OAuth2Adapter`, and implement its methods ([Write an adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#write-an-adapter)). +3. Describe it in `adapter.toml`: its name, protocol, category, and a `[config]` table for each setting ([Describe it](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#describe-it)). +4. Try it with the `relaysms-adapter` console, which runs it the way the Publisher does ([Try it](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#try-it)). +5. Tag a release, such as `v1.0.0` ([Release an adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#release-an-adapter)). +6. Install it on a Publisher: + + ```bash + sudo ./publisher.sh platforms add https://github.com// + ``` + +7. Set its settings with `sudo ./publisher.sh platforms config set =`, or `platforms setup ` if it has a setup ([Settings](../platforms/manage.md#settings)). + +## Check it + +`sudo ./publisher.sh platforms list` shows the adapter with its tag, and ✓ under Configured once its settings are set. Then link a test account from the RelaySMS app. + +The adapters in the [provider guides](../platforms/manage.md#provider-guides), such as [gmail-oauth2-adapter](https://github.com/smswithoutborders/gmail-oauth2-adapter), are complete examples. diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md new file mode 100644 index 0000000..67d8ffa --- /dev/null +++ b/docs/explanation/architecture.md @@ -0,0 +1,71 @@ +# Architecture + +The Publisher is a set of services sharing one database and one Celery broker. Each platform is reached through an adapter that runs as its own process. + +```mermaid +--- +config: + flowchart: + curve: linear +--- +flowchart TB + app["RelaySMS app"] + gateway["Gateway client"] + mailbox["Mailbox"] + platform["Online platforms"] + + subgraph publisher["Publisher"] + grpc["gRPC API"] + rest["REST API"] + smtp["Email listener"] + beat["Celery Beat"] + db[("Database")] + broker[("Celery broker")] + worker["Celery worker"] + adapter["Adapter process"] + end + + app -- "link an account" --> grpc + grpc --> adapter + app -- "SMS" --> gateway + gateway -- "HTTPS" --> rest + gateway -- "email" --> mailbox + mailbox -- "IMAP" --> smtp + rest -- "queue" --> broker + smtp -- "queue" --> broker + beat -- "schedule cleanups" --> broker + broker --> worker + grpc -- "token, keys" --> db + db --> worker + worker --> adapter + adapter --> platform +``` + +Every service in the Publisher also reads and writes the database. + +## Services + +| Service | Does | +| --- | --- | +| gRPC API | Links and unlinks accounts through their adapters, stores their tokens encrypted, and exchanges ephemeral public keys with the app (`SyncKeys`) | +| REST API | Takes payloads from gateway clients, and serves the admin endpoints, publication stats and public lists | +| Email listener | Reads payloads that gateway clients email to a mailbox | +| Celery worker | Decrypts each queued payload and publishes it through the platform's adapter; also installs and updates adapters | +| Celery Beat | Schedules the cleanup jobs and, when set, the worker heartbeat | + +The database holds linked accounts, keys, credentials, adapters, gateway clients, the audit log and publication stats. The broker carries the queue between the services and the worker; with the default SQLite broker it's two files in `data/`. + +## Adapters + +Each adapter is a Python package installed from its own repository into its own virtualenv. The Publisher runs it as a separate process for each call, with a 60-second limit, and talks to it over JSON-RPC on standard input and output. It gets its own config and state directories under `data/platforms/` and a few environment variables, not the Publisher's settings. It runs as the service user ([security model](security-model.md#adapters)). See [Writing an adapter](../developers/writing-an-adapter.md). + +## A publication, step by step + +1. A gateway client posts a payload to `POST /v1/publications`, or emails it. +2. The REST API or the email listener checks its structure, replaces the sender's address with a keyed hash, and queues it. +3. A multi-part payload waits until all its segments arrive. +4. The worker finds the payload's keys, decrypts it, and calls the platform's adapter to publish it. +5. Only a successful publication uses up the keys; a failure leaves them usable. +6. The outcome is recorded in the publication stats. + +The [security model](security-model.md) covers the keys and encryption. diff --git a/docs/explanation/data-retention.md b/docs/explanation/data-retention.md new file mode 100644 index 0000000..8725d91 --- /dev/null +++ b/docs/explanation/data-retention.md @@ -0,0 +1,18 @@ +# Data and retention + +The cleanup jobs run on the [`CELERY_CLEANUP_CRON` and `CELERY_TOKEN_CLEANUP_CRON` schedules](../configuration/settings.md#cleanup-jobs), and delete data past its limit at the next run. + +| Data | Kept until | +| --- | --- | +| Linked accounts' tokens | The user unlinks, or the account has no publication or key sync for `TOKEN_IDLE_MAX_AGE_DAYS` (90); then the Publisher also tries to revoke it on its platform | +| Ephemeral key pools | Each pair is deleted when a publication or token check uses it; the pools go with their token | +| Incomplete multi-part payloads | All segments arrive, or `PAYLOAD_SESSION_MAX_AGE_HOURS` (3) passes | +| Publication stats | Kept | +| Audit events and adapter jobs | `AUDIT_RETENTION_DAYS` (365) | +| Web sessions | Logout, `AUTH_SESSION_IDLE_MINUTES` (30) idle, or `AUTH_SESSION_MAX_HOURS` (12) | +| Credentials, adapters and gateway clients | You remove them | +| Server identity keys | Kept | +| Adapters' own files | Up to each adapter; they live in its state directory | +| Logs | The system journal's retention, and SigNoz's when `OTEL_EXPORTER_OTLP_ENDPOINT` is set | + +See the [security model](security-model.md) for what each record holds. diff --git a/docs/RelaySMS - Technical whitepaper.pdf b/docs/explanation/relaysms-whitepaper.pdf similarity index 100% rename from docs/RelaySMS - Technical whitepaper.pdf rename to docs/explanation/relaysms-whitepaper.pdf diff --git a/docs/explanation/security-model.md b/docs/explanation/security-model.md new file mode 100644 index 0000000..586ec31 --- /dev/null +++ b/docs/explanation/security-model.md @@ -0,0 +1,38 @@ +# Security model + +The RelaySMS protocol is described in the [technical whitepaper](relaysms-whitepaper.pdf). This page covers what the Publisher holds and who can read it. + +## Keys + +| Key | Where | Used for | +| --- | --- | --- | +| Server identity keys | 256 X25519 key pairs in the database | Every gRPC request and every payload; the only key for payloads sent without a linked account | +| Ephemeral key pools | Up to 256 client public keys and 256 server key pairs per linked account | One use each, by a payload or a token check; a successful use deletes the pair | +| [Encryption keys](../configuration/encryption-keys.md) | `.env` | Encrypting the private keys, the platform tokens and, with SQLite, the database file, and keying the sender hash | + +The app refills a linked account's pools with the gRPC `SyncKeys` call. A failed publication leaves its key pair usable, so the same SMS can publish when it's sent again. A client must never encrypt different content with a key pair it has already used. + +## What travels encrypted + +- **Payloads from a linked account** are encrypted by the app with a pool key pair combined with an identity key, so the gateway client and the network carry only ciphertext. +- **Payloads sent without a linked account** are encrypted to an identity key alone, without forward secrecy: anyone who later obtains the identity private keys can decrypt recorded ones. +- **gRPC requests** carry their payload (`x-payload-bin`) encrypted to an identity key. A request is refused if its nonce was already seen, or if its timestamp is more than `NONCE_TTL_SECONDS` from the server's clock. The other request fields, such as phone numbers, login codes and two-step passwords, rely on TLS, from nginx or `GRPC_TLS_ENABLED`. +- **The REST API** takes payloads without credentials, and **the email listener** only from [allowed senders](../configuration/smtp-transport.md#allowlist). Either way, a payload only publishes if it decrypts with a linked account's keys, or passes the [offline publishing](../configuration/offline-publishing.md) rules. + +## What the operator can see + +| Data | Stored as | +| --- | --- | +| Platform tokens | Encrypted with `DATABASE_FIELD_ENCRYPTION_KEY` while `DATABASE_FIELD_ENCRYPTION_ENABLED` is on, as `template.env` sets it | +| Private keys | Encrypted with `DATA_ENCRYPTION_KEY` | +| Sender phone numbers | Replaced with a keyed hash before anything is stored | +| Message content | Ciphertext while queued, and while a multi-part payload waits for its segments; decrypted only in memory, in the worker and the adapter process it hands it to | +| Publication stats | Platform, protocol, status, country code, failure reason and time | + +## Adapters + +An adapter runs as the service user, so it can read `.env` and the database. `PLATFORMS_GITHUB_ORGS` limits which orgs administrators can install from over the REST API. + +## Who can read what + +Anyone holding both `.env` and the database can decrypt the tokens, publish as the linked accounts, and decrypt recorded payloads sent without a linked account. With field encryption off, the database alone exposes the tokens. Protect them as [Backup and restore](../operating/backup-restore.md) and [Security hardening](../operating/security-hardening.md) describe. diff --git a/docs/gateway-clients.md b/docs/gateway-clients.md deleted file mode 100644 index db3a55c..0000000 --- a/docs/gateway-clients.md +++ /dev/null @@ -1,39 +0,0 @@ -# Gateway Clients - -Gateway clients are the phone numbers that relay SMS to this server. They're stored in the `gateway_clients` table and listed publicly at `GET /v1/gateway-clients`; credentials with `gc:read` and `gc:write` manage them over the [REST API](rest.md#managing-gateway-clients). Every change is in the audit log. - -## Commands - -```bash -./publisher.sh gateway-clients suggest # details and PLMN candidates; saves nothing -./publisher.sh gateway-clients create --msisdn --protocols [--country] [--operator] [--operator-code] -./publisher.sh gateway-clients list [--msisdn] [--country] [--operator] -./publisher.sh gateway-clients update [--country] [--operator] [--operator-code] [--protocols] -./publisher.sh gateway-clients disable # hide it from the public list -./publisher.sh gateway-clients enable -./publisher.sh gateway-clients delete -./publisher.sh gateway-clients countries -./publisher.sh gateway-clients operators --country -./publisher.sh gateway-clients mcc-mnc list [--country-code] [--network] [--iso] # search the PLMN table -``` - -`create` fills in what you leave out from `suggest`, the PLMN code only when it's unambiguous. Otherwise pick a candidate and pass it with `--operator-code`. - -## How Suggest Works - -`phonenumbers` gives the number's country, region and carrier name. The carrier is matched against that region's operators in `publisher/gateway_clients/mcc_mnc_table.json`, a vendored snapshot of [musalbas/mcc-mnc-table](https://github.com/musalbas/mcc-mnc-table). A territory without rows of its own, like Guernsey, uses its calling code's main region. - -Names are compared word by word, ignoring case, accents, company-type words ("Telecom", "Ltd", "Mobile") and the country's name, so "MTN Cameroon" matches "MTN" and "Beeline" matches "Bee Line/Unitel". Words many of the region's networks share count only weakly. - -| `match` | Candidates | -| --- | --- | -| `carrier` | Networks matching the carrier, best first. The PLMN code is filled in only when they all share one. | -| `region` | Every operator in the country, because the carrier is unknown (common for US and Canadian numbers) or matched nothing | -| `none` | None: the number couldn't be placed. Enter the details yourself. | - -> [!TIP] -> A PLMN missing from the snapshot can still be entered with `--operator-code`. - -## Upgrading From the JSON Registry - -Migration 019 imports `data/gateway_clients/registry.json` (or `GATEWAY_CLIENTS_REGISTRY_FILE`, if `.env` still sets it). MCC/MNC overrides are no longer used; the migration logs any it finds instead. Both files can then be deleted. diff --git a/docs/gateway-clients/connect-a-relay.md b/docs/gateway-clients/connect-a-relay.md new file mode 100644 index 0000000..2593db9 --- /dev/null +++ b/docs/gateway-clients/connect-a-relay.md @@ -0,0 +1,41 @@ +# Connecting a relay + +A gateway client receives the SMS that people send and relays each payload to the Publisher's `POST /v1/publications`. Pick one: + +| Gateway client | Runs on | Use it for | +| --- | --- | --- | +| [Deku SMS](https://f-droid.org/packages/com.afkanerd.deku/) | An Android phone with a SIM card | A local number, with no SMS provider | +| [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) | A server, behind a Twilio number | A number that doesn't depend on a phone staying online | + +## Requirements + +- The Publisher's REST API reachable over HTTPS, behind the [reverse proxy](../installation/reverse-proxy.md) +- The number [registered as a gateway client](manage.md), so the RelaySMS app lists it + +## Deku SMS on Android + +1. Install [Deku SMS](https://f-droid.org/packages/com.afkanerd.deku/) on the phone and make it the default SMS app. +2. In its gateway settings, forward incoming SMS to `https:///v1/publications`. +3. Keep the phone powered, with mobile data or Wi-Fi. + +## Twilio + +1. Install [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) as its README describes. It can run on the Publisher's server, on its own subdomain such as `twilio.example.com`. Set `TWILIO_AUTH_TOKEN` from the Twilio console, and keep a copy of the generated `SENDER_HASH_KEY`: a new key gives every sender a new address. +2. In its `.env`, set `FORWARD_URLS_JSON=https:///v1/publications`. +3. If the Publisher allows [offline publishing](../configuration/offline-publishing.md) over `https`, set `FORWARD_TAG` to its `OFFLINE_PUBLISH_SHARED_SECRET`. +4. In the Twilio console, under the number's Messaging > A message comes in, set `https:///twilio-sms` with `HTTP POST`. + +## Check it + +Send an SMS from the RelaySMS app to the number, then follow the worker's log on the Publisher: + +```bash +sudo ./manage.sh logs --unit worker +``` + +You should see `Published via ''.` To test without a phone, the [reference client](../developers/reference-client.md#publishing) posts a payload the way a gateway client does. + +## Next steps + +- [Back up](../operating/backup-restore.md) `.env` and `data/`. +- [Harden the install](../operating/security-hardening.md) before announcing the number. diff --git a/docs/gateway-clients/manage.md b/docs/gateway-clients/manage.md new file mode 100644 index 0000000..8e53fd1 --- /dev/null +++ b/docs/gateway-clients/manage.md @@ -0,0 +1,40 @@ +# Managing gateway clients + +Register each number that relays SMS to this server, so the RelaySMS app lists it, then [connect its relay](connect-a-relay.md). Credentials with `gc:read` and `gc:write` can also manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients); every change is recorded in the [audit log](../operating/audit-log.md). + +## Commands + +| Command | Does | +| --- | --- | +| `gateway-clients suggest ` | Shows the number's details and PLMN candidates; saves nothing | +| `gateway-clients create --msisdn --protocols ` | Registers a number; also takes `--country`, `--operator` and `--operator-code` | +| `gateway-clients list` | Lists them; filter with `--msisdn`, `--country` or `--operator` | +| `gateway-clients update ` | Changes `--country`, `--operator`, `--operator-code` or `--protocols` | +| `gateway-clients disable ` | Hides it from the public list | +| `gateway-clients enable ` | Lists it publicly again | +| `gateway-clients delete ` | Removes it | +| `gateway-clients countries` | Lists the countries that have a gateway client | +| `gateway-clients operators --country ` | Lists the operators of that country's gateway clients | +| `gateway-clients mcc-mnc list` | Searches the PLMN table; filter with `--country-code`, `--network` or `--iso` | + +Prefix each with `sudo ./publisher.sh`. `` is how the relay reaches the Publisher, comma-separated: `https` for one that posts to the REST API, `smtp` for one that emails. For example: + +```bash +sudo ./publisher.sh gateway-clients create --msisdn +237600000000 --protocols https +``` + +`create` fills in any of `--country`, `--operator` and `--operator-code` you leave out with what `suggest` finds. It fills the PLMN code (MCC+MNC) only when every candidate shares it; otherwise run `suggest`, pick a candidate and pass it with `--operator-code`. + +## How suggest works + +`phonenumbers` gives the number's country, region and carrier name. The carrier is matched against that region's operators in `publisher/gateway_clients/mcc_mnc_table.json`, a vendored snapshot of [musalbas/mcc-mnc-table](https://github.com/musalbas/mcc-mnc-table). A territory without rows of its own, like Guernsey, uses its calling code's main region. + +Names are compared word by word, ignoring case, accents, the country's name and words such as "Telecom" or "Ltd". + +| `match` | Candidates | +| --- | --- | +| `carrier` | Networks matching the carrier, best first. The PLMN code is filled in only when they all share one. | +| `region` | Every operator in the country, because the carrier is unknown (common for US and Canadian numbers) or matched nothing | +| `none` | None: the number couldn't be placed. Enter the details yourself. | + +When `suggest` can't place the operator, as for many US, Canadian and Twilio numbers, pass `--country` and `--operator` yourself, or leave the operator out. The public list shows each number's country and operator. diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md new file mode 100644 index 0000000..dd8bd04 --- /dev/null +++ b/docs/getting-started/quick-start.md @@ -0,0 +1,78 @@ +# Quick start + +Runs the Publisher on your machine with Docker, to try it before installing it on a server. + +## Requirements + +- **Docker with the Compose plugin**: [Docker Engine](https://docs.docker.com/engine/install/) on Linux, or [Docker Desktop](https://docs.docker.com/desktop/), which includes Compose, on macOS and Windows. On Linux, [add yourself to the `docker` group](https://docs.docker.com/engine/install/linux-postinstall/) or prefix the `docker` commands with `sudo`. +- **Git, `make` and `openssl`**: on macOS, `xcode-select --install` provides Git and `make`, and `openssl` ships with the system. On Windows, run the steps in [WSL](https://learn.microsoft.com/en-us/windows/wsl/install). On Ubuntu or Debian: + + ```bash + sudo apt-get install -y git make openssl + ``` + +Check them: + +```bash +docker compose version +git --version +make --version +openssl version +``` + +## Steps + +1. Clone the repository with its submodule: + + ```bash + git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git + cd RelaySMS-Publisher + ``` + +2. Create `.env` with fresh encryption keys: + + ```bash + make .env + ``` + +3. Build and start. The first build compiles the payload library in Rust and takes several minutes: + + ```bash + docker compose up -d --build + ``` + +4. Check that it's up: + + ```bash + curl http://localhost:16000/health + ``` + + You should see `{"status":"ok"}`. If the connection fails, the container is still applying migrations; `docker compose logs -f` shows its progress. + +5. Create an administrator credential: + + ```bash + docker compose exec publisher \ + python -m publisher creds create --username admin --administrator + ``` + + > [!IMPORTANT] + > The password is printed only once. + +6. Use it on the REST API: + + ```bash + curl -u admin: http://localhost:16000/v1/auth/me + ``` + + You should see `"username":"admin"` and `"administrator":true`. + +To stop the Publisher, run `docker compose down`. To start again from scratch, also run `sudo rm -rf data .env`. + +## Next steps + +The pages below run the admin CLI as `sudo ./publisher.sh`; with Docker, use `docker compose exec publisher python -m publisher` instead. + +- [Add a platform](../platforms/manage.md) +- [Register a gateway client](../gateway-clients/manage.md) +- [Install on a server](../installation/server.md) for production diff --git a/docs/grpc.md b/docs/grpc.md deleted file mode 100644 index 9d5fa9c..0000000 --- a/docs/grpc.md +++ /dev/null @@ -1,399 +0,0 @@ -# Publisher gRPC Documentation - -## Table of Contents - -- [Download Protocol Buffer Files](#download-protocol-buffer-files) -- [Prerequisites](#prerequisites) -- [Version 3 API](#version-3-api) - - [v3: Get OAuth2 Authorization URL](#v3-get-oauth2-authorization-url) - - [v3: Exchange OAuth2 Code and Store Token](#v3-exchange-oauth2-code-and-store-token) - - [v3: Revoke and Delete OAuth2 Token](#v3-revoke-and-delete-oauth2-token) - - [v3: Get PNBA Code](#v3-get-pnba-code) - - [v3: Exchange PNBA Code and Store Token](#v3-exchange-pnba-code-and-store-token) - - [v3: Revoke and Delete PNBA Token](#v3-revoke-and-delete-pnba-token) - - [v3: Sync Keys](#v3-sync-keys) - -## Download Protocol Buffer Files - -### Version 3 - -```bash -curl -O -L https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/staging/protos/v3/publisher.proto -``` - -**Package:** `publisher.v3` -**Service:** `Publisher` - -## Prerequisites - -### Install Dependencies - -```bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -``` - -For other languages, see [Supported languages](https://grpc.io/docs/languages/). - -### Compile gRPC - -```bash -python -m grpc_tools.protoc -I protos/v3 --python_out=. --grpc_python_out=. protos/v3/publisher.proto -``` - -### Starting the Server - -```bash -GRPC_PORT= \ -GRPC_HOST= \ -python3 -m publisher.api.grpc -``` - ---- - -## Version 3 API - -**Package:** `publisher.v3` -**Service:** `Publisher` - -### v3 Request Headers - -All v3 RPC calls require the following headers: - -| Header | Type | Required | Description | -| :--- | :--- | :--- | :--- | -| `x-payload-bin` | bytes | Yes | Encrypted request payload ciphertext (binary) | -| `x-public-key-bin` | bytes | Yes | Client ephemeral public key ($eC_{pk}$, 32 bytes, binary) | -| `x-key-id` | string | Yes | Server static key ID used for encrypting the payload | -| `x-nonce-bin` | bytes | Yes | Request nonce (16 bytes, binary) | -| `x-timestamp` | string | Yes | Unix timestamp in seconds | - -> [!NOTE] -> Headers ending in `-bin` are treated as binary metadata and expect raw binary data. - ---- - -### v3: Get OAuth2 Authorization URL - -Generates an OAuth2 authorization URL for initiating the OAuth2 flow. - -**Request:** `GetOAuth2AuthorizationUrlRequest` - -| Field | Type | Required | Description | -| :--- | :--- | :--- | :--- | -| platform | string | Yes | Platform identifier (e.g., "gmail") | -| state | string | Optional | State parameter to prevent CSRF attacks | -| code_verifier | string | Optional | Cryptographic random string for PKCE | -| autogenerate_code_verifier | bool | Optional | Auto-generate code verifier if not provided | -| redirect_url | string | Optional | Redirect URL for OAuth2 application | -| request_identifier | string | Optional | Request identifier for tracking | - -**Response:** `GetOAuth2AuthorizationUrlResponse` - -| Field | Type | Description | -| :--- | :--- | :--- | -| authorization_url | string | Generated authorization URL | -| state | string | State parameter from request | -| code_verifier | string | Code verifier used in PKCE flow | -| message | string | Response message | -| scope | string | Authorization request scope | -| client_id | string | OAuth2 application client ID | -| redirect_url | string | OAuth2 application redirect URL | - -**Example:** - -```bash -grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ - -d '{ - "platform": "gmail", - "autogenerate_code_verifier": true - }' \ - -proto protos/v3/publisher.proto \ - : publisher.v3.Publisher/GetOAuth2AuthorizationUrl -``` - ---- - -### v3: Exchange OAuth2 Code and Store Token - -Exchanges an OAuth2 authorization code for tokens and stores them. Returns encrypted token data and a set of server ephemeral keys. - -**Request:** `ExchangeOAuth2CodeAndStoreRequest` - -| Field | Type | Required | Description | -| :--- | :--- | :--- | :--- | -| platform | string | Yes | Platform identifier | -| authorization_code | string | Yes | OAuth2 authorization code | -| code_verifier | string | Optional | Code verifier for PKCE | -| redirect_url | string | Optional | Redirect URL for OAuth2 application | -| request_identifier | string | Optional | Request identifier for tracking | -| client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 ephemeral public keys generated by the client, with key_id 0-255 each used once | - -**PublicKey Message:** - -| Field | Type | Description | -| :--- | :--- | :--- | -| key_id | uint32 | Key index (0-255) | -| public_key | bytes | 32-byte public key | - -**Response:** `ExchangeOAuth2CodeAndStoreResponse` - -| Field | Type | Description | -| :--- | :--- | :--- | -| success | bool | Operation success | -| message | string | Response message | -| account_identifier | string | Account identifier (e.g., ) | -| token_ciphertext | bytes | Encrypted token ciphertext | -| token_id | uint32 | Unique identifier for the stored token | -| server_ephemeral_public_keys | list[PublicKey] | Exactly 256 ephemeral public keys generated by the server | - -**Example:** - -```bash -grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ - -d @ \ - -proto protos/v3/publisher.proto \ - : publisher.v3.Publisher/ExchangeOAuth2CodeAndStore <' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ - -d '{ - "token_id": 2941967295, - "key_id": 123 - }' \ - -proto protos/v3/publisher.proto \ - : publisher.v3.Publisher/RevokeOAuth2Token -``` - ---- - -### v3: Get PNBA Code - -Requests a one-time passcode (OTP) for Phone Number-Based Authentication. - -**Request:** `GetPNBACodeRequest` - -| Field | Type | Required | Description | -| :--- | :--- | :--- | :--- | -| platform | string | Yes | Platform identifier | -| phone_number | string | Yes | Phone number to receive OTP | -| request_identifier | string | Optional | Request identifier for tracking | -| channel | string | Optional | The medium used to send the code (e.g., "whatsapp", "telegram", "signal") | - -**Response:** `GetPNBACodeResponse` - -| Field | Type | Description | -| :--- | :--- | :--- | -| success | bool | Operation success | -| message | string | Response message | -| expires_at | int64 | Optional. Code expiry, Unix epoch seconds | - -**Example:** - -```bash -grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ - -d '{ - "platform": "telegram", - "phone_number": "+1234567890" - }' \ - -proto protos/v3/publisher.proto \ - : publisher.v3.Publisher/GetPNBACode -``` - ---- - -### v3: Exchange PNBA Code and Store Token - -Exchanges a one-time passcode (OTP) for a token, stores it, and returns encrypted token data. - -**Request:** `ExchangePNBACodeAndStoreRequest` - -| Field | Type | Required | Description | -| :--- | :--- | :--- | :--- | -| platform | string | Yes | Platform identifier | -| phone_number | string | Yes | Phone number that received OTP | -| authorization_code | string | Yes | PNBA authorization code | -| password | string | Optional | Password for two-step verification | -| request_identifier | string | Optional | Request identifier for tracking | -| client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 ephemeral public keys generated by the client, with key_id 0-255 each used once | -| channel | string | Optional | The medium used to receive the code (e.g., "whatsapp", "telegram", "signal") | - -**Response:** `ExchangePNBACodeAndStoreResponse` - -| Field | Type | Description | -| :--- | :--- | :--- | -| success | bool | Operation success | -| message | string | Response message | -| two_step_verification_enabled | bool | Two-step verification status | -| account_identifier | string | Account identifier | -| token_ciphertext | bytes | Encrypted token ciphertext | -| token_id | uint32 | Unique identifier for the stored token | -| server_ephemeral_public_keys | list[PublicKey] | Exactly 256 ephemeral public keys generated by the server | -| key_id | uint32 | Key ID used for encrypting token | - -**Example:** - -```bash -grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ - -d @ \ - -proto protos/v3/publisher.proto \ - : publisher.v3.Publisher/ExchangePNBACodeAndStore <' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ - -d '{ - "token_id": 2941967295, - "key_id": 123 - }' \ - -proto protos/v3/publisher.proto \ - : publisher.v3.Publisher/RevokePNBAToken -``` - ---- - -### v3: Sync Keys - -Syncs the client and server ephemeral key pools for a token. It rotates the client's key pool with a new set of 256 client ephemeral public keys, and the server generates a new pool of 256 server ephemeral keys. - -**Request:** `SyncKeysRequest` - -| Field | Type | Required | Description | -| :--- | :--- | :--- | :--- | -| token_id | uint32 | Yes | Unique identifier for the stored token | -| key_id | uint32 | Yes | The current key index used for encrypting the token | -| client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 new ephemeral public keys generated by the client, with key_id 0-255 each used once | - -**Response:** `SyncKeysResponse` - -| Field | Type | Description | -| :--- | :--- | :--- | -| success | bool | Operation success | -| message | string | Response message | -| server_ephemeral_public_keys | list[PublicKey] | Exactly 256 ephemeral public keys generated by the server | - -**Example:** - -```bash -grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ - -d @ \ - -proto protos/v3/publisher.proto \ - : publisher.v3.Publisher/SyncKeys < [!NOTE] -> Every RPC returns a standard [gRPC status code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html); `0 OK` is success. diff --git a/docs/installation/celery-broker.md b/docs/installation/celery-broker.md new file mode 100644 index 0000000..faad0a3 --- /dev/null +++ b/docs/installation/celery-broker.md @@ -0,0 +1,47 @@ +# Celery broker + +The REST API and the email listener queue publications for the Celery worker through a broker. + +| `CELERY_BROKER_TYPE` | Use it for | Worker concurrency by default | +| --- | --- | --- | +| `sqlite` (default) | Light load. Needs no server. | 1 | +| `rabbitmq` | Heavier load | 4 | +| `redis` | Heavier load, with a Redis server you already run | 4 | + +`CELERY_WORKER_CONCURRENCY` overrides the concurrency. + +## RabbitMQ + +A setup script installs RabbitMQ, creates a vhost and user, and writes the broker settings to `.env`. + +During install: + +```bash +sudo ./install.sh --setup-broker rabbitmq +``` + +On an existing install, from the install directory: + +```bash +sudo ./scripts/setup-rabbitmq.sh --broker-vhost relaysms --broker-user relaysms +``` + +`--broker-password ` sets the password instead of generating one. To use a broker you already have, add `--existing` (`--broker-existing` on `install.sh`) with `--broker-host`. The script checks the credentials against the management API (port `15672`, or `--broker-mgmt-port`) and changes nothing on the broker. + +## By hand + +For RabbitMQ, in `.env`: + +```bash +CELERY_BROKER_TYPE=rabbitmq +CELERY_RABBITMQ_URL=amqp://relaysms:@localhost:5672/relaysms +``` + +For Redis: + +```bash +CELERY_BROKER_TYPE=redis +CELERY_REDIS_URL=redis://localhost:6379/0 +``` + +Then restart the services with `sudo ./manage.sh restart`. diff --git a/docs/installation/database.md b/docs/installation/database.md new file mode 100644 index 0000000..4b272df --- /dev/null +++ b/docs/installation/database.md @@ -0,0 +1,75 @@ +# Database + +| Dialect | Use it for | Encryption at rest | +| --- | --- | --- | +| SQLite (default) | One server with light load | SQLCipher, with `DATABASE_ENCRYPTION_KEY` | +| MySQL 8.0.28+ or MariaDB | Heavier load, or a database on another host | The server's own (TDE) | +| PostgreSQL 12+ | Heavier load, or a database on another host | The disk's | + +In every dialect, the server's private keys are encrypted, and so are platform tokens while field encryption is on; see [Encryption keys](../configuration/encryption-keys.md). + +## Database servers + +A setup script installs the server, creates the database and user, and writes the connection details to `.env`. Running it again without `--db-password` gives the user a new password. On Debian, install MariaDB yourself and pass `--db-existing`. + +During install: + +```bash +sudo ./install.sh --setup-db postgres +``` + +On an existing install, from the install directory: + +```bash +sudo ./scripts/setup-postgres.sh --db-name relaysms --db-user relaysms +``` + +For MySQL or MariaDB, use `--setup-db mysql` or `scripts/setup-mysql.sh` the same way. + +`--db-password ` sets the password instead of generating one. + +To use a server you already have, pass its details. The script only checks the connection and writes `.env`; it never changes the server: + +```bash +sudo ./install.sh --setup-db postgres --db-existing \ + --db-host db.example.com --db-port 5432 \ + --db-name relaysms --db-user relaysms --db-password '' +``` + +On an existing install: + +```bash +sudo ./scripts/setup-postgres.sh --existing \ + --db-host db.example.com --db-name relaysms --db-user relaysms --db-password '' +``` + +Or set it by hand in `.env`. For PostgreSQL: + +```bash +DATABASE_DIALECT=postgres +POSTGRES_HOST=127.0.0.1 +POSTGRES_PORT=5432 +POSTGRES_USER=relaysms +POSTGRES_PASSWORD= +POSTGRES_DATABASE=relaysms +``` + +For MySQL or MariaDB: + +```bash +DATABASE_DIALECT=mysql +MYSQL_HOST=127.0.0.1 +MYSQL_PORT=3306 +MYSQL_USER=relaysms +MYSQL_PASSWORD= +MYSQL_DATABASE=relaysms +``` + +Then apply the migrations and restart: + +```bash +sudo ./manage.sh migrate +sudo ./manage.sh restart +``` + +The new database starts empty: switching doesn't copy the data from SQLite. diff --git a/docs/installation/docker.md b/docs/installation/docker.md new file mode 100644 index 0000000..d7b16ee --- /dev/null +++ b/docs/installation/docker.md @@ -0,0 +1,22 @@ +# Install with Docker + +`docker-compose.yml` runs every service in one container, with its data in `./data` and its settings in `.env` (see the [settings reference](../configuration/settings.md)). It needs the tools in the [quick start's requirements](../getting-started/quick-start.md#requirements). + +```bash +git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git +cd RelaySMS-Publisher +make .env +docker compose up -d --build +``` + +The container applies pending migrations, then starts every service. It publishes `PORT` (16000) and `GRPC_PORT` (6000) on every interface. To serve it publicly, bind them to `127.0.0.1` in `docker-compose.yml` and put [nginx with TLS](reverse-proxy.md#by-hand) in front. + +Run the admin CLI with `docker compose exec publisher python -m publisher `. + +[Back up](../operating/backup-restore.md) `.env` and `./data`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. + +To update, see [Upgrading](../operating/upgrading.md#update-with-docker). + +## Next steps + +Continue with [After installing](server.md#after-installing) from step 3, running the CLI as `docker compose exec publisher python -m publisher`. diff --git a/docs/installation/manual.md b/docs/installation/manual.md new file mode 100644 index 0000000..50c8f4b --- /dev/null +++ b/docs/installation/manual.md @@ -0,0 +1,86 @@ +# Install by hand + +These are the steps the [installer](server.md) runs, for when you need to change one. Run them in a root shell (`sudo -i`). + +1. Install the system packages: + + ```bash + apt-get update + apt-get install -y python3 python3-pip python3-venv python3-dev \ + build-essential pkg-config libsqlcipher-dev libmagic1 openssl git make curl + ``` + +2. Install Rust, which builds the payload library: + + ```bash + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source "$HOME/.cargo/env" + ``` + +3. Create the service user: + + ```bash + useradd --system --no-create-home --shell /usr/sbin/nologin relaysms + ``` + +4. Clone the repository with its submodule: + + ```bash + git clone --recurse-submodules \ + https://github.com/smswithoutborders/RelaySMS-Publisher.git \ + /opt/relaysms/relaysms-publisher + cd /opt/relaysms/relaysms-publisher + ``` + +5. Create the virtualenv and build the gRPC code and payload library: + + ```bash + python3 -m venv venv + venv/bin/pip install --upgrade pip + venv/bin/pip install -r requirements.txt + make build + ``` + +6. Create `.env` with fresh encryption keys, then set what you need (see the [settings reference](../configuration/settings.md)): + + ```bash + make .env + chown root:relaysms .env + chmod 640 .env + ``` + +7. Create the data directories. By default the SQLite database, Celery files and adapters all live under `data/`: + + ```bash + mkdir -p data/platforms/{adapters,venvs,config,state} + chown -R relaysms: data + chmod -R 750 data + ``` + + If `.env` moves any of these paths out of `data/` (`SQLITE_DATABASE_PATH`, `CELERY_BROKER_DB_PATH`, `CELERY_RESULT_DB_PATH`, `CELERY_BEAT_SCHEDULE_PATH` and the four `PLATFORMS_ADAPTERS_*` directories), create its parent directory and add it to `RW_PATHS` in step 9. Everything else is read-only to the services. + + For MySQL, PostgreSQL or RabbitMQ, run its setup script now ([Database](database.md), [Celery broker](celery-broker.md)). + +8. Check the configuration and apply the database migrations, as the service user: + + ```bash + sudo -u relaysms venv/bin/python -m publisher config check + sudo -u relaysms venv/bin/python -m alembic upgrade head + ``` + +9. Install and start the systemd units. `RW_PATHS` lists the directories the services may write to: + + ```bash + RW_PATHS="$(pwd)/data" + for unit in deploy/systemd/*; do + sed -e "s|/opt/relaysms/relaysms-publisher|$(pwd)|g" \ + -e "s|__RW_PATHS__|$RW_PATHS|" \ + "$unit" > "/etc/systemd/system/$(basename "$unit")" + done + systemctl daemon-reload + systemctl enable --now relaysms-publisher.target + ``` + +## Next steps + +Set up [nginx and TLS](reverse-proxy.md#by-hand), then continue with [After installing](server.md#after-installing). diff --git a/docs/installation/multiple-instances.md b/docs/installation/multiple-instances.md new file mode 100644 index 0000000..d1829bd --- /dev/null +++ b/docs/installation/multiple-instances.md @@ -0,0 +1,21 @@ +# Multiple instances + +To run a second Publisher on the same host, give it its own install directory, instance name and ports: + +1. Install it without nginx. When it shares a database or RabbitMQ server with the first instance, give it its own user: pass `--db-name acme --db-user acme` or `--broker-vhost acme --broker-user acme`. + + ```bash + sudo ./install.sh --install-dir /srv/relaysms-acme --instance-name acme --skip-nginx + ``` + +2. Set `PORT` and `GRPC_PORT` in `/srv/relaysms-acme/.env` to ports no other instance uses. + +3. Restart it, then put nginx in front: + + ```bash + cd /srv/relaysms-acme + sudo ./manage.sh restart + sudo ./manage.sh nginx acme.example.com + ``` + +Its units are named after the instance (`relaysms-publisher-acme.target`, `relaysms-publisher-acme-rest.service`, ...), and the `manage.sh` in each install directory manages only its own instance. Running `install.sh` again on that directory reuses the name; a different `--instance-name` is refused. diff --git a/docs/installation/requirements.md b/docs/installation/requirements.md new file mode 100644 index 0000000..d9ed2c4 --- /dev/null +++ b/docs/installation/requirements.md @@ -0,0 +1,10 @@ +# Requirements + +- **Operating system**: Ubuntu 24.04 or Debian 13, or any Linux host with Docker. +- **Python**: 3.12 or newer. The installer installs it, with the other system packages and Rust. +- **Database**: SQLite (the default) or a database server. See [Database](database.md) for the supported ones. +- **Domain**: a DNS name pointing at the server, for the TLS certificate. See [Reverse proxy and TLS](reverse-proxy.md). +- **Inbound ports (firewall)**: 80 and 443 for nginx. On a server install, the services listen on `127.0.0.1` only. Docker publishes `PORT` (16000) and `GRPC_PORT` (6000). +- **Outbound access**: `github.com`, `pypi.org`, the Rust installer and the system's package mirrors to install the Publisher and its adapters; Let's Encrypt for certificates; each platform's API to publish. +- **Platform credentials**: from each platform you offer, such as a Google Cloud OAuth client for Gmail. See [Provider guides](../platforms/manage.md#provider-guides). +- **Gateway client**: at least one phone number that relays SMS, such as an Android phone running Deku SMS. See [Connecting a relay](../gateway-clients/connect-a-relay.md). diff --git a/docs/installation/reverse-proxy.md b/docs/installation/reverse-proxy.md new file mode 100644 index 0000000..32798be --- /dev/null +++ b/docs/installation/reverse-proxy.md @@ -0,0 +1,43 @@ +# Reverse proxy and TLS + +The services listen on `127.0.0.1`. nginx serves them on port 443 with a Let's Encrypt certificate, from [the site template](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/deploy/nginx/relaysms-publisher-nginx.conf.template). It sends `/publisher.v3.Publisher` and `/grpc.health.v1.Health` to gRPC on `GRPC_PORT`, and everything else to REST on `PORT`. It rate-limits logins and HTTP Basic requests ([limits](../operating/credentials.md#logging-in)). + +## With the installer + +Pass `--site-name ` to the [installer](server.md#run-the-installer). Running it again leaves an existing site file alone, and skips Certbot when the domain already has a certificate; use [`manage.sh nginx`](#on-an-existing-install) to render the site again. + +## On an existing install + +`manage.sh nginx` renders the site from the template, obtains or reattaches its certificate, and turns on HTTP/2, which gRPC needs. Run it to add nginx later, or after an update that changes the template. It keeps the previous file as `.conf.bak` and restores it if the new one fails. + +```bash +sudo apt-get install -y nginx certbot python3-certbot-nginx +sudo ./manage.sh nginx publisher.example.com +``` + +## By hand + +With nginx and Certbot installed as above: + +```bash +sudo sed -e "s/__SERVER_NAME__/publisher.example.com/g" \ + -e "s/__REST_PORT__/16000/g" -e "s/__GRPC_PORT__/6000/g" \ + deploy/nginx/relaysms-publisher-nginx.conf.template | + sudo tee /etc/nginx/sites-available/publisher.example.com.conf >/dev/null +sudo ln -s /etc/nginx/sites-available/publisher.example.com.conf /etc/nginx/sites-enabled/ +sudo nginx -t && sudo systemctl reload nginx +sudo certbot --nginx -d publisher.example.com --redirect +``` + +Then turn on HTTP/2 in the HTTPS block Certbot added: add `http2 on;` after `listen 443 ssl;` (nginx 1.25.1 or newer), or change it to `listen 443 ssl http2;`. Reload nginx. + +## gRPC TLS without nginx + +The gRPC server can serve TLS itself. The REST API can't, so it still needs a reverse proxy. + +```bash +GRPC_HOST=0.0.0.0 +GRPC_TLS_ENABLED=true +GRPC_TLS_CERT_FILE=/path/to/fullchain.pem +GRPC_TLS_KEY_FILE=/path/to/privkey.pem +``` diff --git a/docs/installation/server.md b/docs/installation/server.md new file mode 100644 index 0000000..1fe26ea --- /dev/null +++ b/docs/installation/server.md @@ -0,0 +1,103 @@ +# Install on a server + +The installer sets up the Publisher on Ubuntu 24.04 or Debian 13 under systemd, in `/opt/relaysms/relaysms-publisher`. Check the [requirements](requirements.md) first. + +## Run the installer + +> [!TIP] +> For a dedicated service user with no login, run the installer from a root shell (`sudo -i`). The service user can't be changed later without reinstalling. + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash +``` + +To read the script before running it: + +```bash +curl -fsSLO https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh +less install.sh +sudo bash install.sh +``` + +It asks about each option you don't pass as a flag. To choose up front, pass flags after `bash -s --`: + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash -s -- --site-name publisher.example.com --setup-db postgres +``` + +| Flag | Does | +| --- | --- | +| `--site-name ` | Puts nginx with a Let's Encrypt certificate in front ([details](reverse-proxy.md)) | +| `--letsencrypt-email ` | Email for Let's Encrypt renewal notices | +| `--docs-site-name ` | Serves this manual on its own domain ([details](#serving-the-manual)) | +| `--skip-nginx` | Skips nginx and certificates entirely | +| `--setup-db mysql\|postgres` | Uses MySQL or PostgreSQL instead of SQLite ([details](database.md)) | +| `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker ([details](celery-broker.md)) | +| `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](../operating/observability.md)) | +| `--install-dir `, `--instance-name ` | Installs elsewhere, or a second copy on the same host ([details](multiple-instances.md)) | +| `--branch ` | Installs another branch than `main` | + +`install.sh --help` lists every flag, including those for an existing database or broker. + +The installer: + +1. installs the system packages and Rust; +2. picks the service user: the user who ran `sudo`, or a new `relaysms` user when run from a root shell; +3. clones the repository, builds it and creates `.env` with fresh encryption keys; +4. sets up the database and broker you chose, and applies the database migrations; +5. installs and starts the systemd services; +6. sets up nginx and the certificate, if you gave a domain. + +Running it again updates the install in place and keeps its `.env`. Run it as the same user as the first time; it refuses another service user. + +## After installing + +1. Go to the install directory and check that the services are up: + + ```bash + cd /opt/relaysms/relaysms-publisher + sudo ./manage.sh status + curl http://127.0.0.1:16000/health + ``` + + You should see `{"status":"ok"}`, also at `https:///health` once nginx is set up. + +2. The defaults suit one server behind nginx. Change `.env` only for what they don't cover, such as a MySQL or PostgreSQL database ([settings reference](../configuration/settings.md)), then check it and restart: + + ```bash + sudo ./manage.sh check + sudo ./manage.sh restart + ``` + +3. Create an administrator credential: + + ```bash + sudo ./publisher.sh creds create --username ops --administrator + ``` + + > [!IMPORTANT] + > The password is printed only once. + +4. [Back up](../operating/backup-restore.md) `.env`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. + +## Serving the manual + +`--docs-site-name` builds this manual into `book/` and serves it with nginx on its own domain and certificate. `manage.sh update` rebuilds it. + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash -s -- --site-name publisher.example.com --docs-site-name docs.publisher.example.com +``` + +On an install that's already running, add it from the install directory: + +```bash +sudo ./manage.sh docs docs.publisher.example.com +``` + +## Next steps + +- [Add a platform adapter](../platforms/manage.md), then follow its [provider guide](../platforms/manage.md#provider-guides). +- [Register a gateway client](../gateway-clients/manage.md) and [connect its relay](../gateway-clients/connect-a-relay.md). diff --git a/docs/openapi.json b/docs/openapi.json index 434a82f..b659ca3 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -848,7 +848,7 @@ "Audit" ], "summary": "List audit events", - "description": "Newest first. Scope: audit:read, plus creds:read or platforms:read per area.", + "description": "Newest first. Scope: audit:read, plus each area's read scope.", "operationId": "list_audit_events_v1_audit_events_get", "security": [ { @@ -1507,7 +1507,7 @@ "Publishing" ], "summary": "Publish", - "description": "Queues an SMS payload for publishing.\n\nMulti-part payloads are kept until every segment arrives, then published. Payloads\nare tagged with protocol `https`; see Offline Publishing in the README for\n`OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors\nhappen later in the queue, so they're logged, not returned.", + "description": "Queues an SMS payload for publishing.\n\nMulti-part payloads are kept until every segment arrives, then published. Payloads\nare tagged with protocol `https`; see Offline publishing in the manual for\n`OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors\nhappen later in the queue, so they're logged, not returned.", "operationId": "create_publications_v1_publications_post", "requestBody": { "content": { diff --git a/docs/operating/audit-log.md b/docs/operating/audit-log.md new file mode 100644 index 0000000..d41db03 --- /dev/null +++ b/docs/operating/audit-log.md @@ -0,0 +1,30 @@ +# Audit log + +The Publisher records every change to credentials, adapters and gateway clients, every web login and every refused login, whether made over the REST API or the CLI. Read it at `GET /v1/audit-events`: + +```bash +curl -u ops: https:///v1/audit-events +``` + +It needs `audit:read`, plus the read scope of each area: `creds:read` for `auth.*` and `creds.*` events, `platforms:read` for `platforms.*`, `gc:read` for `gateway_clients.*`. Events from areas you can't read are left out. Filter with `action`, `actor`, `target`, `since` and `until`, such as `?action=auth.login&since=2026-10-01`. Results are [paginated](../reference/rest-api.md#pagination). + +## Events + +| Action | Recorded when | +| --- | --- | +| `creds.create`, `creds.update`, `creds.reset_password`, `creds.revoke_sessions`, `creds.delete` | A credential changes. `details` says what changed, never secrets. | +| `auth.login` | A web session starts, or a web or HTTP Basic login for an existing username is refused | +| `auth.logout` | A web session ends | +| `platforms.add`, `platforms.update`, `platforms.remove` | An adapter is installed, updated or removed. `details` has the source URL and commits. | +| `platforms.enable`, `platforms.disable` | An adapter is offered to users again, or hidden from them | +| `platforms.configure`, `platforms.setup` | An adapter's settings change. `details` has the keys saved. | +| `gateway_clients.create`, `gateway_clients.update`, `gateway_clients.delete` | A gateway client changes. `details` has its fields, or what changed. | +| `gateway_clients.enable`, `gateway_clients.disable` | A gateway client is listed publicly again, or hidden | + +## Fields + +- `outcome`: `success`, `denied` (the change went beyond the actor's scopes) or `failed` (a refused login for an existing username: wrong password, or disabled). +- `actor`: the username at the time; null for the CLI or a failed login. +- `details`: what changed, never secrets; for `auth.login`, `method` is `session` or `basic`. + +Successful HTTP Basic requests aren't recorded. Events are kept for `AUDIT_RETENTION_DAYS` ([Retention](../configuration/settings.md#retention)). diff --git a/docs/operating/backup-restore.md b/docs/operating/backup-restore.md new file mode 100644 index 0000000..0089f0f --- /dev/null +++ b/docs/operating/backup-restore.md @@ -0,0 +1,91 @@ +# Backup and restore + +A backup is: + +- **`.env`**: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. +- **`data/`**: the SQLite database, Celery files, and the adapters with their settings, state and virtualenvs. +- **A dump of the database**, with MySQL or PostgreSQL. +- **Any file `.env` points to outside `data/`**, such as `GRPC_TLS_CERT_FILE` or `SMTP_TLS_CLIENT_KEY`. + +The installer recreates the service units, and the nginx sites and certificates for the domains you pass it, so `/etc` needs no backup. If you run the [Twilio relay](../gateway-clients/connect-a-relay.md#twilio), back up its `.env` too: its `SENDER_HASH_KEY` keeps senders' hashes the same. + +Copy the archives off the host, and test a restore on a spare machine now and then. + +> [!CAUTION] +> Store the `.env` archive apart from the data. Anyone with both can read the linked accounts' tokens. + +## Back up + +From the install directory, stop the services so the data is consistent, then archive `.env` and `data/` separately: + +```bash +sudo ./manage.sh stop +sudo tar -czf /root/relaysms-env-$(date +%F).tar.gz .env +sudo tar -czf /root/relaysms-data-$(date +%F).tar.gz data +``` + +With PostgreSQL, dump the database before starting the services again: + +```bash +sudo sh -c 'sudo -u postgres pg_dump relaysms > /root/relaysms-db-$(date +%F).sql' +``` + +With MySQL or MariaDB: + +```bash +sudo sh -c 'mysqldump relaysms > /root/relaysms-db-$(date +%F).sql' +``` + +Then start the services: + +```bash +sudo ./manage.sh start +``` + +For a database on another host, or named differently in `.env`, connect with the details from `.env`, such as `pg_dump -h -U `. With Docker, use `docker compose stop` and `docker compose start` in place of `manage.sh`. + +## Restore + +1. [Install](../installation/server.md) the Publisher in the same directory, with the same `--instance-name`. With MySQL or PostgreSQL, pass `--setup-db`, `--db-name`, `--db-user` and `--db-password` with the values from the backed-up `.env` (`MYSQL_*` or `POSTGRES_*`); with RabbitMQ, the same for `--broker-password`. Otherwise the servers refuse the restored `.env`'s logins. +2. Stop the services: + + ```bash + sudo ./manage.sh stop + ``` + +3. From the install directory, replace the new `data/` and `.env` with the backed-up ones: + + ```bash + sudo rm -rf data + sudo tar -xzf /root/relaysms-env-.tar.gz + sudo tar -xzf /root/relaysms-data-.tar.gz + ``` + +4. With PostgreSQL, replace the new database with the dump: + + ```bash + sudo -u postgres dropdb relaysms + sudo -u postgres createdb -O relaysms relaysms + sudo sh -c 'sudo -u postgres psql -v ON_ERROR_STOP=1 relaysms < /root/relaysms-db-.sql' + ``` + + With MySQL or MariaDB: + + ```bash + sudo mysql -e 'DROP DATABASE relaysms; CREATE DATABASE relaysms CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' + sudo sh -c 'mysql relaysms < /root/relaysms-db-.sql' + ``` + +5. Apply any migrations newer than the backup, then start the services and check them: + + ```bash + sudo ./manage.sh migrate + sudo ./manage.sh start + sudo ./manage.sh check + ``` + +`tar` keeps the files' owners. If the service user's name or ID differs on the new host, give the files back to it: `sudo chown -R : data` and `sudo chown root: .env`. + +## Next steps + +After a restore, [check that the services are up](services.md#is-it-working). diff --git a/docs/operating/credentials.md b/docs/operating/credentials.md new file mode 100644 index 0000000..abe4d2a --- /dev/null +++ b/docs/operating/credentials.md @@ -0,0 +1,54 @@ +# Credentials and access + +Credentials log in to the REST API's admin endpoints, each with its own scopes. One holding every scope is an administrator. The RelaySMS app and gateway clients need none. + +## Create a credential + +```bash +sudo ./publisher.sh creds create --username ops --administrator +sudo ./publisher.sh creds create --username analyst --scope stats:publications:read +``` + +> [!IMPORTANT] +> The password is printed only once. + +Usernames are 3 to 32 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit. + +## Scopes + +| Scope | Allows | +| --- | --- | +| `stats:publications:read` | Read publication stats, without failure reasons | +| `stats:publications:reasons` | See failure reasons in publication stats | +| `gc:read` | List gateway clients | +| `gc:write` | Add, change and remove gateway clients | +| `platforms:read` | List platform adapters | +| `platforms:write` | Change and remove platform adapters | +| `creds:read` | List credentials | +| `creds:write` | Add, change and remove credentials | +| `audit:read` | Read the [audit log](audit-log.md), for the areas your other scopes cover | + +`sudo ./publisher.sh creds scopes` prints the same list. Each `:write` scope needs its `:read` scope, and `stats:publications:reasons` needs `stats:publications:read`. Installing, updating and configuring adapters over the API also needs an administrator. + +## Commands + +| Command | Does | +| --- | --- | +| `creds list` | Lists credentials | +| `creds set-scopes --username --scope ` | Replaces a credential's scopes; repeat `--scope`, or pass `--administrator` | +| `creds reset-password --username ` | Prints a new password and ends the credential's web sessions | +| `creds disable --username ` | Blocks it and ends its web sessions | +| `creds enable --username ` | Lets it log in again | +| `creds revoke-sessions --username ` | Logs it out of every web session; HTTP Basic still works | +| `creds delete --username ` | Deletes it and its sessions, after asking; `--yes` skips the question | + +Prefix each with `sudo ./publisher.sh`. Credentials with `creds:write` can do the same over the [REST API](../reference/rest-api.md#managing-credentials), within their own scopes. + +## Logging in + +- **Web session:** `POST /v1/auth/login`, then send the session cookie with every request (`credentials: "include"` in `fetch`). Logins and writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from the API's own origin or one listed in `AUTH_WEB_ORIGINS`, on the same site as the API. A session ends after `AUTH_SESSION_IDLE_MINUTES` idle (30) or `AUTH_SESSION_MAX_HOURS` (12). +- **HTTP Basic:** the username and password on every request, such as `curl -u analyst: https:///v1/stats/publications`. + +Scope changes apply on the credential's next request. nginx allows 5 logins a minute per IP address, and 30 HTTP Basic requests a minute per IP address. + +Settings: [REST API access](../configuration/settings.md#rest-api-access). diff --git a/docs/operating/observability.md b/docs/operating/observability.md new file mode 100644 index 0000000..fa97842 --- /dev/null +++ b/docs/operating/observability.md @@ -0,0 +1,181 @@ +# Observability + +Self-hosted tracing, metrics and log export (SigNoz) and uptime monitoring (Uptime Kuma) for the Publisher's services. Both are optional; tracing, metrics and log export stay off while `OTEL_EXPORTER_OTLP_ENDPOINT` is blank. + +## Set up + +During install: + +```bash +sudo ./install.sh --setup-observability +``` + +On an existing install, from the install directory: + +```bash +sudo ./scripts/setup-observability.sh +``` + +| `install.sh` | `setup-observability.sh` | Does | +| --- | --- | --- | +| `--observability-site-name ` | `--site-name ` | Puts nginx and a Let's Encrypt certificate in front of SigNoz | +| `--observability-kuma-site-name ` | `--kuma-site-name ` | The same for Uptime Kuma, on a different domain | +| None | `--signoz-email`, `--signoz-password` | SigNoz admin account | +| None | `--signoz-smtp-host`, `--signoz-smtp-port`, `--signoz-smtp-username`, `--signoz-smtp-password`, `--signoz-smtp-from` | SMTP relay for SigNoz's email alerts | +| None | `--skip-tracing` | Leaves tracing off | + +`scripts/setup-observability.sh --help` lists every flag. The script: + +- installs Docker if it's missing; +- writes `observability/signoz/casting.yaml` from its template, binds the SigNoz UI and OTLP ports to `127.0.0.1`, starts SigNoz and creates the admin account; +- starts Uptime Kuma on `127.0.0.1:3001`, or the port you choose; +- installs `requirements-observability.txt` and sets `OTEL_EXPORTER_OTLP_ENDPOINT`, unless given `--skip-tracing`; +- with `--site-name` or `--kuma-site-name`, writes and enables the nginx sites and runs Certbot. + +Set retention in the SigNoz UI, under Settings > Workspace > Retention Controls. + +> [!IMPORTANT] +> `casting.yaml` is generated only once. To change a running install, edit `observability/signoz/casting.yaml` and run `foundryctl cast -f observability/signoz/casting.yaml` again. It recreates only the `signoz` container; Postgres and ClickHouse keep their data. + +## Uptime Kuma monitors + +Open Uptime Kuma through its domain, or through an SSH tunnel at : + +```bash +ssh -L 3001:127.0.0.1:3001 @ +``` + +Then add a monitor for each service. + +### REST + +Monitor Type "HTTP(s)", URL `https:///health`. + +### gRPC + +1. Choose Monitor Type "GRPC(S) - Keyword". +2. Set gRPC URL to `:443` and turn on Enable TLS. +3. Set Service Name to `grpc.health.v1.Health` and Method to `check`. +4. Set gRPC Body to `{"service":"publisher.v3.Publisher"}` and Keyword to `SERVING`. +5. Paste this into Protobuf: + +
+Health check protobuf + +```proto +syntax = "proto3"; +package grpc.health.v1; + +message HealthCheckRequest { + string service = 1; +} + +message HealthCheckResponse { + enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; + SERVICE_UNKNOWN = 3; + } + ServingStatus status = 1; +} + +service Health { + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); +} +``` + +
+ +### Worker + +Monitor Type "Push", interval just over 60s. Set `UPTIME_KUMA_WORKER_PUSH_URL` in `.env` to the push URL Kuma shows, then restart the services. Beat schedules this heartbeat, so the monitor also goes down when Beat stops. + +### SMTP listener + +As for the worker, with an interval just over 20s and `UPTIME_KUMA_SMTP_PUSH_URL`. + +## Alerts + +In the SigNoz UI, go to Alerts > New Alert > Logs, paste a filter below into the search bar, set the threshold to count > 0 over a rolling window, and attach a notification channel. Keep the parentheses: `AND` binds tighter than `OR`. + +Publish failures, adapter errors included: + +``` +service.name = 'relaysms-publisher-worker' AND (body CONTAINS 'Failed to process payload' OR body CONTAINS 'Failed to publish message' OR body CONTAINS 'An unexpected error occurred during task processing') +``` + +Key management failures: + +``` +body CONTAINS 'Failed to generate server identity keys' OR body CONTAINS 'Token hash missing' +``` + +## Turning tracing on and off + +To turn tracing on by hand, from the install directory: + +```bash +sudo venv/bin/pip install --quiet -r requirements-observability.txt +sudo sed -i 's|^OTEL_EXPORTER_OTLP_ENDPOINT=.*|OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317|' .env +sudo ./manage.sh restart +``` + +To turn it off, blank `OTEL_EXPORTER_OTLP_ENDPOINT` and restart. + +## Setting up by hand + +### SigNoz + +Needs Docker Engine with the Compose plugin. Set `` to 12 or more characters, with upper and lower case, a number and a symbol. + +```bash +curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_INSTALL_DIR=/usr/local/bin bash + +sed \ + -e "s/__POSTGRES_DB__/signoz/" -e "s/__POSTGRES_USER__/signoz/" \ + -e "s/__POSTGRES_PASSWORD__/$(openssl rand -hex 24)/" \ + -e "s/__SIGNOZ_PORT__/8090/" -e "s/__OTLP_GRPC_PORT__/4317/" -e "s/__OTLP_HTTP_PORT__/4318/" \ + -e "s/__SMTP_SMARTHOST__//" -e "s/__SMTP_USERNAME__//" -e "s/__SMTP_PASSWORD__//" -e "s/__SMTP_FROM__//" \ + -e "s|__SIGNOZ_EXTERNAL_URL__|http://localhost:8090|" \ + observability/signoz/casting.yaml.template > observability/signoz/casting.yaml + +foundryctl cast -f observability/signoz/casting.yaml +docker compose \ + -f pours/deployment/compose.yaml \ + -f observability/signoz/docker-compose.override.yml \ + up -d + +until curl -fs http://127.0.0.1:8090/api/v1/version | grep -q setupCompleted; do sleep 2; done +curl -X POST http://127.0.0.1:8090/api/v1/register \ + -H "Content-Type: application/json" \ + -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@example.com","password":""}' +``` + +The template stores SigNoz's metadata in Postgres (`metastore.kind: postgres`); set `sqlite` for a single node. + +### Uptime Kuma + +```bash +docker compose -f observability/uptime-kuma/docker-compose.yml up -d +``` + +### Reverse proxy + +```bash +sed -e "s/__SERVER_NAME__//g" \ + observability/nginx-observability.conf.template | + sudo tee /etc/nginx/sites-available/.conf >/dev/null +sudo ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ + +sed -e "s/__SERVER_NAME__//g" \ + observability/nginx-uptime-kuma.conf.template | + sudo tee /etc/nginx/sites-available/.conf >/dev/null +sudo ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ + +sudo nginx -t && sudo systemctl reload nginx +sudo certbot --nginx -d +sudo certbot --nginx -d +``` + +`` serves SigNoz and `` serves Uptime Kuma. diff --git a/docs/operating/security-hardening.md b/docs/operating/security-hardening.md new file mode 100644 index 0000000..b83a69e --- /dev/null +++ b/docs/operating/security-hardening.md @@ -0,0 +1,35 @@ +# Security hardening + +A server install listens on `127.0.0.1` only, runs each service under systemd sandboxing with everything read-only except its data directories, and sits behind nginx with TLS and rate limits. These steps close what's left to you. + +## Network and firewall + +1. Allow only SSH, HTTP and HTTPS in: + + ```bash + sudo ufw allow OpenSSH + sudo ufw allow 'Nginx Full' + sudo ufw enable + ``` + +2. Keep `HOST` and `GRPC_HOST` at `127.0.0.1`, so only nginx reaches the services. +3. On Docker, bind `PORT` and `GRPC_PORT` to `127.0.0.1` in `docker-compose.yml`, since Docker's published ports bypass ufw, and put nginx in front ([details](../installation/docker.md)). + +## Service user + +The services run as the user who ran the installer with `sudo`, or as a new `relaysms` user when it ran from a root shell. For a dedicated user with no login or `sudo` rights, run the installer from a root shell (`sudo -i`). Changing it on an existing install means reinstalling. + +## Settings + +| Setting | Keep it at | +| --- | --- | +| `API_DOCS_ENABLED` | `false` in production | +| `AUTH_SESSION_COOKIE_SECURE` | `true` | +| `AUTH_WEB_ORIGINS` | Only the web client origins you run | +| `PLATFORMS_GITHUB_ORGS` | Only orgs you trust: installing an adapter runs its code on the server | +| `SMTP_ALLOWED_SENDERS` | Exact addresses; a domain trusts every sender on it ([details](../configuration/smtp-transport.md#allowlist)) | +| `DATABASE_ENCRYPTION_ENABLED`, `DATABASE_FIELD_ENCRYPTION_ENABLED` | `true` | + +## Credentials + +Give each person their own [credential](credentials.md), with only the scopes they need. Review the [audit log](audit-log.md) for `denied` events (a change beyond the actor's scopes) and `failed` ones (a refused login). diff --git a/docs/operating/services.md b/docs/operating/services.md new file mode 100644 index 0000000..9799aea --- /dev/null +++ b/docs/operating/services.md @@ -0,0 +1,82 @@ +# Services + +The Publisher runs as systemd services, grouped under `relaysms-publisher.target`: + +| Service | Runs | +| --- | --- | +| `relaysms-publisher-rest` | REST API | +| `relaysms-publisher-grpc` | gRPC API | +| `relaysms-publisher-worker` | Celery worker: publishes queued payloads | +| `relaysms-publisher-beat` | Celery Beat: schedules cleanups, and the worker heartbeat with Uptime Kuma | +| `relaysms-publisher-smtp` | [Email listener](../configuration/smtp-transport.md); exits at once while `SMTP_TRANSPORT_ENABLED` is off | + +The REST API listens on `HOST:PORT` (`127.0.0.1:16000`) and gRPC on `GRPC_HOST:GRPC_PORT` (`127.0.0.1:6000`). + +## Is it working? + +```bash +sudo ./manage.sh status +curl http://127.0.0.1:16000/health +``` + +You should see every service active, and `{"status":"ok"}`. To follow publications as they happen: + +```bash +sudo ./manage.sh logs --unit worker +``` + +Each published payload logs `Published via ''.`, and each failure a line starting `Failed to` ([what each means](troubleshooting.md#a-payload-is-accepted-but-never-published)). Credentials with `stats:publications:read` can also query the counts at `GET /v1/stats/publications`. + +## manage.sh + +Run it from the install directory with `sudo`: + +| Command | Does | +| --- | --- | +| `status` | Shows each service's state | +| `logs` | Follows the service logs | +| `start`, `stop`, `restart` | Controls every service together | +| `enable`, `disable` | Starts the services on boot, or stops doing so | +| `update [--migrate]` | Pulls the latest code, rebuilds and restarts the services ([details](upgrading.md)) | +| `migrate` | Applies database migrations | +| `check` | Reports every missing or invalid setting | +| `nginx []` | Renders the nginx site again and reattaches or obtains its certificate ([details](../installation/reverse-proxy.md#on-an-existing-install)) | +| `docs []` | Builds this manual and serves it on its own domain ([details](../installation/server.md#serving-the-manual)) | +| `uninstall` | Removes the services and the install directory, with `.env` and `data/` | + +`uninstall` keeps the database and broker servers and the service user. + +`logs` takes: + +- `-u`, `--unit`: `rest`, `grpc`, `worker`, `beat` or `smtp`; repeatable +- `-n`, `--lines`: how many lines to show +- `-s`, `--since`: a time, such as `"1 hour ago"` +- `--no-follow`: print and exit + +For example: + +```bash +sudo ./manage.sh logs --unit rest --unit grpc --since "1 hour ago" --lines 200 +``` + +## publisher.sh + +`./publisher.sh` runs the admin CLI as the service user, with the virtualenv and `.env`: + +```bash +sudo ./publisher.sh --help # every command +sudo ./publisher.sh env # install directory, service user and the .env values the CLI uses +sudo ./publisher.sh shell # a shell as the service user, with .env loaded +``` + +Use it instead of `python3 -m publisher`: run as another user, the CLI can leave adapter files the services can't read. + +## Files + +| Path | Holds | +| --- | --- | +| `/opt/relaysms/relaysms-publisher/` | The install | +| `.env` | Settings (`root:`, mode 640) | +| `data/` | SQLite database, Celery files and adapters (owned by the service user, mode 750) | +| `/etc/systemd/system/relaysms-publisher*` | Service units | +| `/etc/nginx/sites-available/.conf` | nginx sites | diff --git a/docs/operating/troubleshooting.md b/docs/operating/troubleshooting.md new file mode 100644 index 0000000..e3ac2ea --- /dev/null +++ b/docs/operating/troubleshooting.md @@ -0,0 +1,101 @@ +# Troubleshooting + +Start with the logs of the service involved: + +```bash +sudo ./manage.sh logs --unit --since "1 hour ago" +``` + +`` is one of the [units `logs` takes](services.md#managesh). + +## Services + +### A service won't start + +The service refuses to start on an invalid setting it uses. Run `sudo ./manage.sh check`, fix what it reports in `.env`, then `sudo ./manage.sh restart`. + +### `update` ends with `.env has errors` + +The new version needs a setting `.env` lacks. Run `sudo ./sync-env.sh`, fill in what `sudo ./manage.sh check` reports, then `sudo ./manage.sh restart`. + +### `/health` returns `500` + +The REST API can't open the database: the database server is down, its settings in `.env` are wrong, or, with SQLite, the service user can't read `data/`. See [Database](../installation/database.md). + +### The email listener service is inactive + +It exits at once while `SMTP_TRANSPORT_ENABLED` is off. See [SMTP transport](../configuration/smtp-transport.md). + +### Docker refuses connections right after starting + +The container applies migrations before the services start. Wait, or follow `docker compose logs -f`. + +## Publishing + +### A payload is accepted but never published + +`POST /v1/publications` only queues a payload; the worker decides later. Follow `sudo ./manage.sh logs --unit worker` and match the line: + +| Log line | Cause | +| --- | --- | +| `Published via ''.` | It worked | +| `Failed to process payload: ...` | The payload didn't decrypt or validate, its keys are unknown or used up, or offline publishing refused it ([Offline publishing](../configuration/offline-publishing.md)) | +| `Failed to publish message: Adapter error: ...` | The platform refused it; the rest of the line is the adapter's reason. Fix it with the platform's [provider guide](../platforms/manage.md#provider-guides), then the user sends the same SMS again. | +| `Session can't be assembled from segment(s): ...` | A multi-part payload's segments don't fit together | +| `An unexpected error occurred during task processing.` | A bug; the traceback follows. [Report it](https://github.com/smswithoutborders/RelaySMS-Publisher/issues). | +| No line at all | The worker isn't running (`sudo ./manage.sh status`), or a multi-part payload is still waiting for its other segments, for up to `PAYLOAD_SESSION_MAX_AGE_HOURS` | + +`GET /v1/stats/publications` lists each failure with a reason code, for credentials with `stats:publications:reasons`. + +### Linking an account fails + +Check the adapter shows ✓ under Configured in `sudo ./publisher.sh platforms list`, then see its [provider guide](../platforms/manage.md#provider-guides)'s troubleshooting. + +## Relay + +### An SMS was sent, but the worker logs nothing + +Check that the relay received it: + +- With Deku SMS, the phone is online and Deku SMS is its default SMS app. +- With Twilio, follow the relay's log (`sudo journalctl -u relaysms-gatewayclient-twilio`) and Twilio's error log, under Monitor > Logs > Errors in the Twilio console. + +### The Twilio relay answers `403` `Invalid Twilio signature.` + +The webhook URL in the Twilio console doesn't match the URL the relay sees, scheme and host included. Set it to the relay's exact `https://` URL. + +### The Twilio relay answers `502` `Forwarding failed.` + +A `FORWARD_URLS_JSON` URL refused the payload or couldn't be reached; the relay's log names it. Check that `https:///v1/publications` answers from the relay's host. + +## REST API + +### `Invalid username or password.` + +The credential doesn't exist, the password is wrong, or it's disabled (`sudo ./publisher.sh creds list`). + +### `Missing scope: ...` + +Add the scope with `sudo ./publisher.sh creds set-scopes` ([Scopes](credentials.md#scopes)). + +### `Session expired or invalid. Please log in again.` + +The session idled past `AUTH_SESSION_IDLE_MINUTES`, reached `AUTH_SESSION_MAX_HOURS`, or was ended by `reset-password`, `disable` or `revoke-sessions`. Log in again. + +### `Origin not allowed.` + +The web client's origin is neither the API's own nor listed in `AUTH_WEB_ORIGINS`; logins and writes with a session cookie both check it. Add the origin, which must be on the same site as the API, then `sudo ./manage.sh restart`. + +### `Too many requests. Please try again shortly.` + +nginx rate-limits logins and HTTP Basic requests ([limits](credentials.md#logging-in)). Wait a minute and try again. + +## nginx + +### gRPC calls fail through nginx + +gRPC needs HTTP/2 on the HTTPS block. Run `sudo ./manage.sh nginx` to render the site again with it. + +### `nginx -t` fails after an update + +Run `sudo ./manage.sh nginx`: it renders the site from the current template, and restores the previous file if the new one fails. diff --git a/docs/operating/upgrading.md b/docs/operating/upgrading.md new file mode 100644 index 0000000..4bef4aa --- /dev/null +++ b/docs/operating/upgrading.md @@ -0,0 +1,86 @@ +# Upgrading + +A server install follows the branch it was installed from, `main` by default. + +## Before you upgrade + +1. [Back up](backup-restore.md). Migrations can't be undone, so going back to older code needs the backup. +2. From the install directory, note the current commit, to roll back to: + + ```bash + sudo git rev-parse HEAD + ``` + +## Update a server install + +From the install directory: + +```bash +sudo ./manage.sh update --migrate +sudo ./sync-env.sh +sudo ./manage.sh check +sudo ./manage.sh restart +``` + +1. `update --migrate` stops the services, pulls the code, reinstalls the dependencies, rebuilds, applies pending migrations and starts the services again. When the install serves this manual, it rebuilds it too. +2. `sync-env.sh` adds the settings the new `template.env` has and `.env` lacks, showing the change before writing it. +3. `check` reports any setting still missing or invalid; fill it in `.env`. +4. `restart` loads the new settings. + +When `.env` has an invalid setting, `update` still restarts every service, then exits with an error ([fix](troubleshooting.md#update-ends-with-env-has-errors)). + +If the update changed the nginx template, which `sudo git diff --stat ORIG_HEAD HEAD -- deploy/nginx` lists, render the site again: + +```bash +sudo ./manage.sh nginx +``` + +If you edited tracked files that the update changes, `update` refuses to pull. Discard the edits or `git stash` them, then run `update` again. + +## Roll back + +1. Take the commit you noted [before upgrading](#before-you-upgrade). +2. If the update applied migrations, [restore the backup](backup-restore.md#restore) taken before it; older code can't use the migrated database. +3. Check out that commit and rebuild. `--pulled` skips the pull that would undo the checkout: + + ```bash + sudo git checkout + sudo ./manage.sh update --pulled + ``` + +To follow the branch again later, run `sudo git checkout `, then `sudo ./manage.sh update --migrate`. + +## Update with Docker + +```bash +git pull --recurse-submodules +docker compose up -d --build +``` + +The container applies new migrations and runs `platforms import` when it starts. + +## Update adapters + +Adapters update separately, by version tag: + +```bash +sudo ./publisher.sh platforms update +``` + +See [Versions](../platforms/manage.md#versions). + +## Upgrade notes + +Steps a particular upgrade needs once, newest first. + +### Gateway client registry import (migration 019) + +For installs from before 2026-10-07. + +`update --migrate` applies migration 019, which imports `data/gateway_clients/registry.json` (or `GATEWAY_CLIENTS_REGISTRY_FILE`, if `.env` sets it). It logs the entries in `mcc_mnc_overrides.json` without importing them. Delete both files afterwards. + +### Adapter registry import + +For installs from before 2026-10-06. + +`update --migrate` and `manage.sh migrate` register adapter directories on disk that the database lacks, and move their settings and files out of the code. The Docker container does it at every start. diff --git a/docs/platforms.md b/docs/platforms.md deleted file mode 100644 index b04042e..0000000 --- a/docs/platforms.md +++ /dev/null @@ -1,77 +0,0 @@ -# Platform Adapters - -Each platform (Gmail, X, Telegram, ...) is served by an adapter installed from its own GitHub repository. Adapters are registered in the `platform_adapters` table; their files live under `data/platforms/` (`PLATFORMS_ADAPTERS_*` in `.env`). - -> [!IMPORTANT] -> Run these commands with `./publisher.sh` from the install directory, not `python3 -m publisher`. It runs as the service user, so the services can still read the adapter files. - -## Commands - -```bash -./publisher.sh platforms add [--tag TAG] # install the newest, or a given, version tag -./publisher.sh platforms update [NAME] [--tag TAG] # move one adapter, or all, to a version tag -./publisher.sh platforms list # every adapter, disabled ones included -./publisher.sh platforms disable # hide it from users; it still revokes tokens -./publisher.sh platforms enable # offer it again -./publisher.sh platforms remove [--force] # uninstall -./publisher.sh platforms config show # its settings, without secret values -./publisher.sh platforms config set KEY=VALUE|KEY # change settings; a bare KEY asks for the value -./publisher.sh platforms setup [--input KEY=VALUE] # run the adapter's setup and save its settings -./publisher.sh platforms import # register adapter directories, move their files out of the code -./publisher.sh platforms exec -- [ARGS] # run a command the adapter installs -``` - -Commands taking a name also take `--proto-id` and `--cat-id`, for when a name matches more than one adapter. Administrators can do the same over the [REST API](rest.md#managing-platform-adapters), installing only from orgs in `PLATFORMS_GITHUB_ORGS`. - -## Versions - -Adapters are installed and updated by version tag, such as `v1.2.0`, so a repository needs one before it can be added. - -An update builds the new version and its virtualenv beside the running one, then switches over, so a failed update leaves the running version in place. - -> [!NOTE] -> An update is refused if the installed tag now points to a different commit. Release tags shouldn't move. - -## Removing - -> [!WARNING] -> `remove` is refused while accounts are linked through the adapter, since their tokens can only be revoked through it. Disable it instead. `--force` removes it anyway and leaves those tokens unrevocable. - -## Files - -| Directory | Holds | -|---|---| -| `PLATFORMS_ADAPTERS_DIR/` | The adapter's code at its tag, replaced on every update | -| `PLATFORMS_ADAPTERS_VENV_DIR/` | Its virtualenv | -| `PLATFORMS_ADAPTERS_CONFIG_DIR/` | Its `credentials.json` | -| `PLATFORMS_ADAPTERS_STATE_DIR/` | What it writes, such as databases | - -`` is the ID column of `platforms list`. - -## Settings - -An adapter reads its settings, such as API keys, from `credentials.json` in its config directory. Its `adapter.toml` declares each one. Lists are comma-separated. - -```bash -./publisher.sh platforms config set rmail smtp_host=smtp.example.com smtp_port=465 -./publisher.sh platforms config set rmail smtp_password -./publisher.sh platforms config set rmail --unset alias_prefix -``` - -`config set` changes only the keys given. `setup` is for adapters whose settings come from the platform, such as an app it registers or a file its console downloads; it replaces the whole file. `--input KEY=@PATH` reads a file: - -```bash -./publisher.sh platforms setup gmail --input client_file=@client_secret.json -``` - -## An Adapter's Own Commands - -An adapter may install admin commands. `exec` runs one from the adapter's virtualenv with its config and state directories; put `--` before it: - -```bash -./publisher.sh platforms exec -- --help -``` - -## Writing an Adapter - -Build it with the [adapter SDK](../sdk/README.md), then tag a release and `add` it. diff --git a/docs/platforms/bluesky.md b/docs/platforms/bluesky.md new file mode 100644 index 0000000..1f039ca --- /dev/null +++ b/docs/platforms/bluesky.md @@ -0,0 +1,67 @@ +# Bluesky + +Lets users post to Bluesky. Bluesky has no app registration: the Publisher serves the client's metadata at a public URL, and that URL is the client ID. + +*Last checked against the AT Protocol OAuth spec: 2026-10-22.* + +## Requirements + +- The REST API is public over HTTPS on port 443, behind the [reverse proxy](../installation/reverse-proxy.md). +- `https:///v1/platforms/bluesky/oauth/client-metadata.json` answers `200` with JSON, with no redirect on the way. It does while the adapter is enabled. +- The [redirect URIs](manage.md#redirect-uris) use HTTPS. + +## Configure the adapter + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/bluesky-oauth2-adapter +sudo ./publisher.sh platforms config set bluesky \ + client_id=https://publisher.example.com/v1/platforms/bluesky/oauth/client-metadata.json \ + client_uri=https://publisher.example.com \ + client_name=RelaySMS \ + redirect_uris=https://example.com/callback \ + "scope=atproto transition:generic" +``` + +| Setting | Required | Description | +| --- | --- | --- | +| `client_id` | Yes | The metadata URL above, exactly as the Publisher serves it | +| `client_uri` | Yes | Home page on the Publisher's host, the same host as `client_id` | +| `client_name` | Yes | Name on Bluesky's consent page | +| `redirect_uris` | Yes | Comma-separated redirect URIs your apps send, the default first | +| `scope` | Yes | `atproto transition:generic` | +| `pds_url` | No | Auth server. Default: `https://bsky.social` | + +## Check it + +```bash +curl -i https://publisher.example.com/v1/platforms/bluesky/oauth/client-metadata.json +``` + +You should see `200` and JSON whose `client_id` equals the URL you fetched. + +## Troubleshooting + +### Bluesky says the client metadata is invalid + +`client_id` must match the served URL exactly: `https`, `/v1`, no trailing slash. + +### The metadata URL redirects or needs a login + +Bluesky refuses redirects. Serve the URL directly over HTTPS. + +### `Access wasn't granted to every scope.` + +Set `scope` to exactly `atproto transition:generic`. + +### `The account uses a different auth server.` + +The account isn't on `pds_url`'s auth server, such as one on a self-hosted PDS. Only accounts there can link. + +### A changed redirect URI isn't accepted yet + +Bluesky's auth server caches the metadata. Try again later. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/platforms/gmail.md b/docs/platforms/gmail.md new file mode 100644 index 0000000..46db7d9 --- /dev/null +++ b/docs/platforms/gmail.md @@ -0,0 +1,99 @@ +# Gmail + +Lets users send email from their Gmail account. + +*Last checked against Google Cloud Console: 2026-10-22.* + +## Requirements + +- A Google Cloud account +- An HTTPS [redirect URI](manage.md#redirect-uris) + +## Create the OAuth client + +1. [Create a project](https://console.cloud.google.com/projectcreate), or pick one. +2. [Enable the Gmail API](https://console.cloud.google.com/apis/library/gmail.googleapis.com). +3. In Google Auth Platform > [Branding](https://console.cloud.google.com/auth/branding), click **Get started**. +4. Enter the app name, support email and contact email. Choose **External** as the audience, unless every user belongs to your Google Workspace organization. +5. In [Data Access](https://console.cloud.google.com/auth/scopes), add these scopes and save: + - `openid` + - `https://www.googleapis.com/auth/userinfo.email` + - `https://www.googleapis.com/auth/userinfo.profile` + - `https://www.googleapis.com/auth/gmail.send` +6. In [Clients](https://console.cloud.google.com/auth/clients), create a client of type **Web application**. +7. Under Authorized redirect URIs, add every redirect URI your apps send, the default first, and create the client. +8. Download the client's JSON from the dialog that confirms it. + +> [!IMPORTANT] +> Google shows the client secret only at creation. If you close the dialog without downloading the JSON, add a new secret to the client. + +## Configure the adapter + +Copy the JSON to the server, then: + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/gmail-oauth2-adapter +sudo ./publisher.sh platforms setup gmail \ + --input client_file=@/tmp/client_secret_.apps.googleusercontent.com.json +``` + +`setup` copies the client ID, secret and redirect URIs from the file. + +To add a redirect URI later, add it to the client in the console, then set the whole list: + +```bash +sudo ./publisher.sh platforms config set gmail \ + redirect_uris=https://a.example.com/callback,https://b.example.com/callback +``` + +## Check it + +```bash +sudo ./publisher.sh platforms config show gmail +``` + +You should see `redirect_uris` in the client's order, and `client_secret` as `(secret, set)`. + +## Testing and production + +| Publishing status | Who can link | Limits | +| --- | --- | --- | +| Testing | Test users listed under [Audience](https://console.cloud.google.com/auth/audience) | Links expire after 7 days | +| In production, unverified | Anyone, after a warning | 100 users in all | +| In production, verified | Anyone | None | + +`gmail.send` is a sensitive scope, so [verification](https://developers.google.com/identity/protocols/oauth2/production-readiness/sensitive-scope-verification) needs: + +- the redirect and home page domains verified in Google Search Console; +- a public home page and a privacy policy on that domain, describing how the app uses Gmail; +- a justification for each scope; +- an unlisted YouTube video showing the consent screen and how the app sends email. + +Publish the app under Audience, then submit it in the [Verification Center](https://console.cloud.google.com/auth/verification). + +## Troubleshooting + +### `setup` fails with `redirect_uris is required` + +The client has no authorized redirect URIs. Add them in the console, then set them with `config set gmail redirect_uris=...`. + +### `redirect_uri_mismatch` + +The URI the app sent isn't on the client. Add it exactly, trailing slash included. + +### Linking fails with `Access wasn't granted to ...` + +The user unticked a permission on the consent screen. They should link again and allow every permission. + +### Sending fails with `Google refused the refresh token` + +Links made while the app is in Testing expire after 7 days. Publish the app, and have users link again. + +### The client disappeared + +Google deletes clients unused for 6 months. Restore it within 30 days, or create a new one. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/platforms/manage.md b/docs/platforms/manage.md new file mode 100644 index 0000000..eda6e29 --- /dev/null +++ b/docs/platforms/manage.md @@ -0,0 +1,101 @@ +# Managing adapters + +Each platform, such as Gmail, is served by an adapter installed from its own GitHub repository. + +Run these commands from the install directory with [`publisher.sh`](../operating/services.md#publishersh), which runs them as the service user. + +## Commands + +| Command | Does | +| --- | --- | +| `platforms add [--tag ]` | Installs the newest version tag, or `` | +| `platforms update [] [--tag ]` | Moves one adapter, or all, to a version tag | +| `platforms list` | Lists every adapter, disabled ones included | +| `platforms disable ` | Hides it from users; linked accounts can still be unlinked | +| `platforms enable ` | Offers it again | +| `platforms remove [--force]` | Uninstalls it ([details](#removing)) | +| `platforms config show ` | Lists its settings, without secret values | +| `platforms config set =\|` | Changes settings; a bare `` asks for its value | +| `platforms setup [--input =]` | Runs the adapter's setup and saves its settings | +| `platforms import` | Registers adapter directories on disk that the database lacks, and moves their files out of the code ([details](../operating/upgrading.md#adapter-registry-import)) | +| `platforms exec -- ` | Runs a command the adapter installs | + +Prefix each with `sudo ./publisher.sh`, for example: + +```bash +sudo ./publisher.sh platforms add https://github.com/smswithoutborders/gmail-oauth2-adapter +``` + +Commands taking a name also take `--proto-id` and `--cat-id`, for when a name matches more than one adapter. Administrators can do the same over the [REST API](../reference/rest-api.md#managing-platform-adapters), but only from GitHub orgs listed in `PLATFORMS_GITHUB_ORGS`. + +## Versions + +Adapters are installed and updated by version tag, such as `v1.2.0`, so a repository needs one before it can be added. + +An update builds the new version and its virtualenv beside the running one, then switches over, so a failed update leaves the running version in place. An update is refused if the installed tag now points to a different commit: publish a new tag instead of moving one. + +## Removing + +`remove` is refused while accounts are linked through the adapter, since their tokens can only be revoked through it. Disable the adapter instead. + +> [!WARNING] +> `--force` removes it anyway and leaves those tokens unrevocable. + +## Files + +| Directory | Holds | +| --- | --- | +| `PLATFORMS_ADAPTERS_DIR/` | The adapter's code at its tag, replaced on every update | +| `PLATFORMS_ADAPTERS_VENV_DIR/` | Its virtualenv | +| `PLATFORMS_ADAPTERS_CONFIG_DIR/` | Its `credentials.json` | +| `PLATFORMS_ADAPTERS_STATE_DIR/` | What it writes, such as databases | + +`` is the ID column of `platforms list`. + +## Settings + +An adapter reads its settings, such as API keys, from `credentials.json` in its config directory. Its `adapter.toml` declares each one. Lists are comma-separated. + +```bash +sudo ./publisher.sh platforms config set rmail smtp_host=smtp.example.com smtp_port=465 +sudo ./publisher.sh platforms config set rmail smtp_password +sudo ./publisher.sh platforms config set rmail --unset alias_prefix +``` + +`config set` changes only the keys given. `setup` is for adapters whose settings come from the platform, such as a registered app for Mastodon or a downloaded client file for Gmail, and replaces all of them. `--input =@` reads a file as the service user, from the install directory, so give an absolute path it can read: + +```bash +sudo ./publisher.sh platforms setup gmail --input client_file=@/tmp/client_secret.json +``` + +## Redirect URIs + +OAuth2 adapters, such as Gmail, send the user back to a redirect URI after they authorize. The client app chooses it and sends it to the Publisher when it links an account; the adapter uses the first of its `redirect_uris` setting when the app sends none. + +1. Run an HTTPS page that receives the authorization `code` and `state` and hands them back to the app. +2. Configure the apps to send that page's URI. +3. Register the same URI, character for character, with the platform. A trailing slash counts. +4. Set it in the adapter's `redirect_uris`, with every other URI your apps send. + +## Provider guides + +| Platform | Adapter | Protocol | +| --- | --- | --- | +| [Gmail](gmail.md) | [gmail-oauth2-adapter](https://github.com/smswithoutborders/gmail-oauth2-adapter) | OAuth2 | +| [Mastodon](mastodon.md) | [mastodon-oauth2-adapter](https://github.com/smswithoutborders/mastodon-oauth2-adapter) | OAuth2 | +| [Bluesky](bluesky.md) | [bluesky-oauth2-adapter](https://github.com/smswithoutborders/bluesky-oauth2-adapter) | OAuth2 | +| [Telegram](telegram.md) | [telegram-pnba-adapter](https://github.com/smswithoutborders/telegram-pnba-adapter) | Phone number | +| [RelaySMS Mail](relaysms-mail.md) | [relaysms-mail-pnba-adapter](https://github.com/smswithoutborders/relaysms-mail-pnba-adapter) | Phone number | + +## An adapter's own commands + +An adapter may install admin commands. `exec` runs one from the adapter's virtualenv with its config and state directories; put `--` before it: + +```bash +sudo ./publisher.sh platforms exec -- --help +``` + +## Next steps + +- Follow the [provider guide](#provider-guides) for each platform you offer, then link a test account from the RelaySMS app. +- To support another platform, see [Writing an adapter](../developers/writing-an-adapter.md). diff --git a/docs/platforms/mastodon.md b/docs/platforms/mastodon.md new file mode 100644 index 0000000..c854e0c --- /dev/null +++ b/docs/platforms/mastodon.md @@ -0,0 +1,59 @@ +# Mastodon + +Lets users post to Mastodon. The adapter's setup registers the app on a Mastodon server, so there's no console to visit. + +*Last checked against the Mastodon API: 2026-10-22.* + +## Requirements + +- A Mastodon server running 4.4 or newer: see `version` at `https:///api/v2/instance` +- An HTTPS [redirect URI](manage.md#redirect-uris) + +The adapter serves one Mastodon server: only accounts on that server can link. + +## Configure the adapter + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/mastodon-oauth2-adapter +sudo ./publisher.sh platforms setup mastodon \ + --input client_name=RelaySMS \ + --input redirect_uris=https://example.com/callback \ + --input website=https://example.com \ + --input base_url=https://mastodon.social +``` + +| Input | Required | Description | +| --- | --- | --- | +| `client_name` | Yes | App name on the server's consent page | +| `redirect_uris` | Yes | Comma-separated redirect URIs your apps send, the default first | +| `website` | No | Home page on the consent page | +| `base_url` | No | The server. Default: `https://mastodon.social` | + +`setup` registers the app with the scopes `profile`, `write:statuses` and `write:media`, and saves its client ID and secret. Run it again to change the redirect URIs or the server; it registers a new app and replaces every setting. After changing the server, users must link again. + +## Check it + +```bash +sudo ./publisher.sh platforms config show mastodon +``` + +You should see `client_secret` as `(secret, set)`. + +## Troubleshooting + +### A user can't link + +Their account is on another server than `base_url`. + +### Linking fails after authorizing + +The server runs Mastodon older than 4.4. + +### "The redirect URI included is not valid" + +The app sent a URI the app registration doesn't have. Run `setup` again with it. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/platforms/relaysms-mail.md b/docs/platforms/relaysms-mail.md new file mode 100644 index 0000000..273b4c2 --- /dev/null +++ b/docs/platforms/relaysms-mail.md @@ -0,0 +1,92 @@ +# RelaySMS Mail + +The `rmail` adapter gives each user an email alias tied to their phone number, and sends email from it through [SimpleLogin](https://simplelogin.io). Users link with a code sent over a messaging app by [Shortmesh Authy](https://github.com/shortmesh/Authy-API). It's the adapter that publishes [offline payloads](../configuration/offline-publishing.md), each from a new random alias. + +*Last checked against SimpleLogin, Shortmesh Authy and Google: 2026-10-22.* + +## Requirements + +- A SimpleLogin Premium account, which custom domains and sending to new contacts require +- A domain whose DNS you control +- An SMTP account allowed to send as your SimpleLogin mailbox address +- A self-hosted Shortmesh Authy, to send the login codes + +## SimpleLogin + +1. Sign up at [app.simplelogin.io](https://app.simplelogin.io). The signup address becomes your mailbox; to use another, add and verify it under [Mailboxes](https://app.simplelogin.io/dashboard/mailbox). +2. Add your domain, or a subdomain such as `mail.example.com`, under [Custom domains](https://app.simplelogin.io/dashboard/custom_domain). +3. Create the DNS records the domain's page lists: ownership TXT, MX, SPF, DKIM and DMARC. SimpleLogin marks the last three optional; without them, mail sent from aliases doesn't arrive. +4. Create an [API key](https://app.simplelogin.io/dashboard/api_key). + +## SMTP + +The adapter sends each message from your mailbox address over SMTP, and SimpleLogin relays it from the user's alias. + +For a Gmail mailbox, use `smtp.gmail.com`, port `465`, the Gmail address, and an [app password](https://myaccount.google.com/apppasswords), which needs 2-Step Verification on the account. + +## Shortmesh Authy + +Run [Authy-API](https://github.com/shortmesh/Authy-API) with its Interface-API, then in the Interface-API's admin UI: + +1. Under Devices, link the phone that sends the codes. +2. Under Tokens, create a token. + +## Configure the adapter + +This example uses a Gmail mailbox: + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/relaysms-mail-pnba-adapter +sudo ./publisher.sh platforms config set rmail \ + sl_primary_email=you@gmail.com sl_primary_domain=mail.example.com \ + smtp_host=smtp.gmail.com smtp_port=465 smtp_username=you@gmail.com \ + authy_base_url=https://authy.example.com authy_sender=+237600000000 +sudo ./publisher.sh platforms config set rmail sl_api_key smtp_password authy_token +``` + +`sudo ./publisher.sh platforms config show rmail` lists the optional settings, such as alias prefixes. + +## Check it + +```bash +sudo ./publisher.sh platforms list --name rmail +``` + +You should see ✓ under Configured. + +```bash +curl https://authy.example.com/api/v1/platforms +``` + +You should see the channels Authy can send codes on, such as `signal`. + +## Troubleshooting + +### `No mailbox for ` + +`sl_primary_email` isn't a verified SimpleLogin mailbox. + +### `No suffix available for hostname ''` + +`sl_primary_domain` isn't a verified custom domain in SimpleLogin. + +### Sending fails with an SMTP authentication error + +For Gmail, use an app password, not the account password. A changed account password revokes app passwords. + +### Recipients don't get the email + +Check the domain's SPF, DKIM and DMARC records on its SimpleLogin page. + +### SimpleLogin refuses to create a contact + +The account isn't Premium. + +### Users don't get a login code + +Check that the sender device is linked and online, and that the channel the app sends, such as `signal`, appears at `/api/v1/platforms`. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/platforms/telegram.md b/docs/platforms/telegram.md new file mode 100644 index 0000000..8c973ee --- /dev/null +++ b/docs/platforms/telegram.md @@ -0,0 +1,49 @@ +# Telegram + +Lets users send Telegram messages. Users link with their phone number, a login code and, if they set one, their two-step verification password. + +*Last checked against my.telegram.org: 2026-10-22.* + +## Requirements + +- Agreement to Telegram's [API terms of service](https://core.telegram.org/api/terms) +- A Telegram account made with an official app, on a phone number you'll keep. Each number can create only one API ID. + +## Get the API ID and hash + +1. Sign in at [my.telegram.org](https://my.telegram.org) with the number in international format. The login code arrives in the Telegram app, not by SMS. +2. Open **API development tools**. +3. Fill in the app title and a short name (5 to 32 letters and digits), and create the application. +4. Copy the **App api_id** and **App api_hash**. + +The hash can't be changed: replacing a leaked one takes a new app on another number. + +## Configure the adapter + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/telegram-pnba-adapter +sudo ./publisher.sh platforms config set telegram api_id= api_hash +``` + +## Check it + +```bash +sudo ./publisher.sh platforms list --name telegram +``` + +You should see ✓ under Configured. + +## Troubleshooting + +### The form at my.telegram.org says only "ERROR" + +Try again later from a home connection without a VPN, in a private window. + +### A user's account is limited or banned + +Heavy or spam-like sending gets accounts banned. Appeals go to recover@telegram.org. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 0000000..020fe3c --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,31 @@ +# Command-line tools + +| Tool | Runs | Documented in | +| --- | --- | --- | +| `install.sh` | Installs or updates a server install | [Install on a server](../installation/server.md) | +| `manage.sh` | Controls, updates and checks the services | [Services](../operating/services.md#managesh) | +| `publisher.sh` | The admin CLI, as the service user | [Below](#publishersh) | +| `sync-env.sh` | Adds to `.env` the settings `template.env` has and it lacks | [Upgrading](../operating/upgrading.md) | +| `scripts/setup-postgres.sh`, `scripts/setup-mysql.sh` | Sets up a database server | [Database](../installation/database.md) | +| `scripts/setup-rabbitmq.sh` | Sets up a RabbitMQ broker | [Celery broker](../installation/celery-broker.md) | +| `scripts/setup-observability.sh` | Sets up SigNoz and Uptime Kuma | [Observability](../operating/observability.md) | + +Run them with `sudo`; all but `install.sh` from the install directory. Each takes `--help`. + +## manage.sh + +See [Services](../operating/services.md#managesh). + +## publisher.sh + +Runs `python3 -m publisher` as the service user. `sudo ./publisher.sh --help` describes each command. + +| Group | Commands | Documented in | +| --- | --- | --- | +| `creds` | `create`, `list`, `scopes`, `set-scopes`, `reset-password`, `disable`, `enable`, `revoke-sessions`, `delete` | [Credentials and access](../operating/credentials.md) | +| `platforms` | `add`, `update`, `list`, `disable`, `enable`, `remove`, `config show`, `config set`, `setup`, `import`, `exec` | [Managing adapters](../platforms/manage.md) | +| `gateway-clients` | `suggest`, `create`, `list`, `update`, `disable`, `enable`, `delete`, `countries`, `operators`, `mcc-mnc list` | [Managing gateway clients](../gateway-clients/manage.md) | +| `config` | `check` | [Settings reference](../configuration/settings.md) | +| `seed` | `stats`, `creds`, `platforms`, `gateway-clients` | None; fills a development database with sample data | + +It also takes `env` and `shell` ([Services](../operating/services.md#publishersh)). diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md new file mode 100644 index 0000000..aadcbf4 --- /dev/null +++ b/docs/reference/glossary.md @@ -0,0 +1,24 @@ +# Glossary + +- **Adapter** (platform adapter): the plugin that publishes to one platform, such as Gmail, installed from its own repository. See [Managing adapters](../platforms/manage.md). +- **Administrator**: a credential holding every scope. +- **Audit event**: a record of a change or login. See [Audit log](../operating/audit-log.md). +- **Category**: what an adapter publishes, such as email (`--cat-id`). +- **Credential**: a username and password that logs in to the REST API's admin endpoints, with its own scopes. See [Credentials and access](../operating/credentials.md). +- **Ephemeral key pool**: the client public keys and server key pairs each linked account holds, up to 256 of each; each pair is used once, with an identity key, by a payload or a token check. See [Security model](../explanation/security-model.md). +- **Gateway client**: a phone number that relays SMS to the Publisher, such as an Android phone running Deku SMS. See [Connecting a relay](../gateway-clients/connect-a-relay.md). +- **Identity key**: one of the server's 256 long-term X25519 keys, used in every gRPC request and payload, and alone for offline payloads. +- **Linked account**: a user's account on a platform, authorized once while online. The Publisher stores its token encrypted. +- **MSISDN**: a phone number in international format, such as `+237600000000`. +- **OAuth2**: linking through the platform's consent page, as for Gmail. +- **Offline payload**: content sent without a linked account, published through [RelaySMS Mail](../platforms/relaysms-mail.md). See [Offline publishing](../configuration/offline-publishing.md). +- **Payload**: the encrypted content the RelaySMS app sends by SMS; one carrying an attachment arrives in several segments. +- **PLMN**: a mobile network, identified by its MCC (country code) and MNC (network code). See [How suggest works](../gateway-clients/manage.md#how-suggest-works). +- **PNBA** (phone number-based authentication): linking with a code sent to the user's phone, as for Telegram. +- **Protocol**: how an adapter links accounts, such as `oauth2` (`--proto-id`). For a payload, how it reached the Publisher, such as `https` (the REST API); see [Offline publishing](../configuration/offline-publishing.md). +- **Redirect URI**: where an OAuth2 platform sends the user back after they authorize. See [Redirect URIs](../platforms/manage.md#redirect-uris). +- **rmail**: the name of the RelaySMS Mail adapter. See [RelaySMS Mail](../platforms/relaysms-mail.md). +- **Scope**: a permission a credential holds, such as `gc:write`. See [Scopes](../operating/credentials.md#scopes). +- **Sender hash**: the keyed hash that replaces a sender's phone number before anything is stored. +- **Token**: what the Publisher stores for a linked account to act on the platform, encrypted. +- **Web session**: a REST API login kept in a cookie, for a web client. diff --git a/docs/reference/grpc-api.md b/docs/reference/grpc-api.md new file mode 100644 index 0000000..30dbc68 --- /dev/null +++ b/docs/reference/grpc-api.md @@ -0,0 +1,345 @@ +# gRPC API + +The gRPC API links and unlinks platform accounts and syncs ephemeral key pools. The RelaySMS app calls it while online. + +## Protocol buffer file + +```bash +curl -O -L https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/protos/v3/publisher.proto +``` + +Package `publisher.v3`, service `Publisher`. + +## Generate a client + +Generate stubs with `protoc` for your [language](https://grpc.io/docs/languages/). For Python: + +```bash +pip install grpcio-tools +python -m grpc_tools.protoc -I . --python_out=. --grpc_python_out=. publisher.proto +``` + +## Request headers + +Every RPC needs these headers. Headers ending in `-bin` carry raw binary. + +| Header | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| `x-payload-bin` | bytes | Yes | Encrypted request payload ciphertext | +| `x-public-key-bin` | bytes | Yes | Client ephemeral public key ($eC_{pk}$, 32 bytes) | +| `x-key-id` | string | Yes | ID of the server identity key used to encrypt the payload | +| `x-nonce-bin` | bytes | Yes | Request nonce (16 bytes) | +| `x-timestamp` | string | Yes | Unix timestamp in seconds, within `NONCE_TTL_SECONDS` of the server's clock | + +The examples use `-plaintext` against a local server; drop it for a server behind TLS on port 443. + +## Methods + +### Get OAuth2 authorization URL + +Returns the URL that starts the OAuth2 flow. + +**Request:** `GetOAuth2AuthorizationUrlRequest` + +| Field | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| platform | string | Yes | Platform identifier, such as `gmail` | +| state | string | No | OAuth2 `state`, returned unchanged | +| code_verifier | string | No | PKCE code verifier | +| autogenerate_code_verifier | bool | No | Generate a code verifier when none is sent | +| redirect_url | string | No | Redirect URL for OAuth2 application | +| request_identifier | string | No | Request identifier for tracking | + +**Response:** `GetOAuth2AuthorizationUrlResponse` + +| Field | Type | Description | +| :--- | :--- | :--- | +| authorization_url | string | Generated authorization URL | +| state | string | The `state` sent | +| code_verifier | string | PKCE code verifier used | +| message | string | Human-readable result | +| scope | string | Authorization request scope | +| client_id | string | OAuth2 application client ID | +| redirect_url | string | OAuth2 application redirect URL | + +**Example:** + +```bash +grpcurl -plaintext \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ + -d '{ + "platform": "gmail", + "autogenerate_code_verifier": true + }' \ + -proto publisher.proto \ + : publisher.v3.Publisher/GetOAuth2AuthorizationUrl +``` + +### Exchange OAuth2 code and store token + +Exchanges an OAuth2 authorization code for tokens and stores them. Returns encrypted token data and a set of server ephemeral keys. + +**Request:** `ExchangeOAuth2CodeAndStoreRequest` + +| Field | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| platform | string | Yes | Platform identifier | +| authorization_code | string | Yes | OAuth2 authorization code | +| code_verifier | string | No | Code verifier for PKCE | +| redirect_url | string | No | Redirect URL for OAuth2 application | +| request_identifier | string | No | Request identifier for tracking | +| client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 ephemeral public keys generated by the client, with key_id 0-255 each used once | + +**`PublicKey` message:** + +| Field | Type | Description | +| :--- | :--- | :--- | +| key_id | uint32 | Key index (0-255) | +| public_key | bytes | 32-byte public key | + +**Response:** `ExchangeOAuth2CodeAndStoreResponse` + +| Field | Type | Description | +| :--- | :--- | :--- | +| success | bool | `true` if the call succeeded | +| message | string | Human-readable result | +| account_identifier | string | Account identifier, such as `mail@example.com` | +| token_ciphertext | bytes | Encrypted token ciphertext | +| token_id | uint32 | Unique identifier for the stored token | +| server_ephemeral_public_keys | list[PublicKey] | Exactly 256 ephemeral public keys generated by the server | + +**Example:** + +```bash +grpcurl -plaintext \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ + -d @ \ + -proto publisher.proto \ + : publisher.v3.Publisher/ExchangeOAuth2CodeAndStore <' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ + -d '{ + "token_id": 2941967295, + "key_id": 123 + }' \ + -proto publisher.proto \ + : publisher.v3.Publisher/RevokeOAuth2Token +``` + +### Get PNBA code + +Requests a one-time passcode (OTP) for phone number-based authentication (PNBA). + +**Request:** `GetPNBACodeRequest` + +| Field | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| platform | string | Yes | Platform identifier | +| phone_number | string | Yes | Phone number to receive OTP | +| request_identifier | string | No | Request identifier for tracking | +| channel | string | No | Channel to send the code over, such as `signal` | + +**Response:** `GetPNBACodeResponse` + +| Field | Type | Description | +| :--- | :--- | :--- | +| success | bool | `true` if the call succeeded | +| message | string | Human-readable result | +| expires_at | int64 | Optional. Code expiry, Unix epoch seconds | + +**Example:** + +```bash +grpcurl -plaintext \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ + -d '{ + "platform": "telegram", + "phone_number": "+237600000000" + }' \ + -proto publisher.proto \ + : publisher.v3.Publisher/GetPNBACode +``` + +### Exchange PNBA code and store token + +Exchanges a one-time passcode (OTP) for a token, stores it, and returns encrypted token data. + +**Request:** `ExchangePNBACodeAndStoreRequest` + +| Field | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| platform | string | Yes | Platform identifier | +| phone_number | string | Yes | Phone number that received OTP | +| authorization_code | string | Yes | PNBA authorization code | +| password | string | No | Password for two-step verification | +| request_identifier | string | No | Request identifier for tracking | +| client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 ephemeral public keys generated by the client, with key_id 0-255 each used once | +| channel | string | No | Channel the code was sent over, such as `signal` | + +**Response:** `ExchangePNBACodeAndStoreResponse` + +| Field | Type | Description | +| :--- | :--- | :--- | +| success | bool | `true` if the call succeeded | +| message | string | Human-readable result | +| two_step_verification_enabled | bool | Two-step verification status | +| account_identifier | string | Account identifier | +| token_ciphertext | bytes | Encrypted token ciphertext | +| token_id | uint32 | Unique identifier for the stored token | +| server_ephemeral_public_keys | list[PublicKey] | Exactly 256 ephemeral public keys generated by the server | +| key_id | uint32 | Key ID used for encrypting token | + +**Example:** + +```bash +grpcurl -plaintext \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ + -d @ \ + -proto publisher.proto \ + : publisher.v3.Publisher/ExchangePNBACodeAndStore <' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ + -d '{ + "token_id": 2941967295, + "key_id": 123 + }' \ + -proto publisher.proto \ + : publisher.v3.Publisher/RevokePNBAToken +``` + +### Sync keys + +Replaces a token's key pools: the client sends 256 new ephemeral public keys and the server returns 256 of its own. + +**Request:** `SyncKeysRequest` + +| Field | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| token_id | uint32 | Yes | Unique identifier for the stored token | +| key_id | uint32 | Yes | The current key index used for encrypting the token | +| client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 new ephemeral public keys generated by the client, with key_id 0-255 each used once | + +**Response:** `SyncKeysResponse` + +| Field | Type | Description | +| :--- | :--- | :--- | +| success | bool | `true` if the call succeeded | +| message | string | Human-readable result | +| server_ephemeral_public_keys | list[PublicKey] | Exactly 256 ephemeral public keys generated by the server | + +**Example:** + +```bash +grpcurl -plaintext \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ + -d @ \ + -proto publisher.proto \ + : publisher.v3.Publisher/SyncKeys </v1 +``` + +`/health` is the only endpoint outside `/v1`. + +## Authentication + +Admin endpoints take a web session cookie or HTTP Basic credentials; see [Logging in](../operating/credentials.md#logging-in). Each endpoint's description names the scopes it needs. + +## Managing credentials + +A credential can only grant scopes it holds, can't change itself, and can't change a credential holding scopes it lacks. These return `403`. + +`GET /v1/creds/{username}` returns an opaque `ETag`. Send it back unchanged as `If-Match` to change, reset or delete that credential: a missing header gets `428`, a stale one `412`. Responses that change a credential return its new `ETag`. Generated passwords appear only in the response that creates them. + +## Managing platform adapters + +`/v1/platforms/adapters` lists every installed adapter, with its source, version tag, commit and who last changed it, and takes the same `ETag` and `If-Match` rules as credentials. A disabled adapter disappears from `/v1/platforms` and can't publish or link accounts, but still revokes tokens so users can unlink. Uninstalling is refused with `409` while accounts are linked through the adapter; see [Removing](../platforms/manage.md#removing). + +Installing (`POST /v1/platforms/adapters`) and updating (`POST /v1/platforms/adapters/{id}/update`) run code from the repository on the server, so they need an administrator and a repository from an org in `PLATFORMS_GITHUB_ORGS`. Both take an optional `tag`, defaulting to the newest version tag, and return `202` with the job's URL in `Location`. Poll the job until its `state` is `succeeded` or `failed`; its `log` has the end of the git and pip output. An adapter has one job at a time; another gets `409`. + +An adapter's settings are at `/v1/platforms/adapters/{id}/config`, for administrators only. `GET` lists each setting, with secrets only marked `is_set`, and the inputs its setup takes. `PATCH` changes the settings sent and keeps the rest; `POST .../setup` runs the adapter's setup and replaces them all, with a file input sent as the file's text. `configured` in the adapter list says whether every required setting has a value. + +## Managing gateway clients + +`/v1/gateway-clients/registry` lists every gateway client, with who last changed it, and takes the same `ETag` and `If-Match` rules as credentials (scopes `gc:read` and `gc:write`). `GET /v1/gateway-clients/registry/suggest?msisdn=` returns the details and PLMN candidates a create would use, to prefill a form for the administrator to confirm; see [how suggest works](../gateway-clients/manage.md#how-suggest-works). A disabled client disappears from the public `/v1/gateway-clients`. + +## Pagination + +List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. Follow `next` and `prev` as they are: they keep your filters and `limit`, and are `null` on the last and first page. Don't build the `cursor` yourself. + +`since` and `until` take ISO-8601 times, UTC if there's no offset. `since` is inclusive and `until` exclusive. + +## Audit events + +`GET /v1/audit-events` returns the [audit log](../operating/audit-log.md). + +## Publishing + +`POST /v1/publications` takes JSON and tags payloads with protocol `https`. It needs no credentials: a payload publishes only if it decrypts with the keys of a linked account or, for an offline payload, passes the [offline publishing](../configuration/offline-publishing.md) rules. + +It only checks the payload's structure before queueing it. Decryption, payload type and adapter errors happen later, so they're logged on the server, not returned. + +## Errors + +Error bodies are `{"error": ""}`. The message is meant to be shown to users; the server logs more detail. + +| Status | Meaning | +| :--- | :--- | +| `400 Bad Request` | Invalid parameters or payload, invalid cursor, or invalid time window | +| `401 Unauthorized` | Missing or invalid credentials | +| `403 Forbidden` | Origin not allowed, missing scope, or a credential change beyond your scopes | +| `404 Not Found` | The resource doesn't exist | +| `409 Conflict` | Already exists, in use, or changed by another request at the same moment; retry | +| `412 Precondition Failed` | `If-Match` doesn't match the current `ETag`; reload and retry | +| `422 Unprocessable Entity` | Validation error | +| `428 Precondition Required` | `If-Match` header missing | +| `429 Too Many Requests` | Rate limited by nginx ([limits](../operating/credentials.md#logging-in)) | +| `500 Internal Server Error` | Unexpected server error | diff --git a/docs/rest.md b/docs/rest.md deleted file mode 100644 index de725d1..0000000 --- a/docs/rest.md +++ /dev/null @@ -1,91 +0,0 @@ -# Publisher REST API - -The REST API serves platform, gateway client and server key metadata, accepts encrypted payloads for publishing, and lets credentials read stats, manage credentials and read the audit log. - -The endpoint reference is [openapi.json](openapi.json), generated from the app. Open it in any OpenAPI viewer, or set `API_DOCS_ENABLED=true` on a dev server and browse `/docs`. After changing an endpoint, run `make docs`; a test fails while the file is stale. - -This page covers what the reference doesn't show well. - -## Base URL - -``` -https:///v1 -``` - -`/health` is the only endpoint outside `/v1`. - -## Authentication - -Credentials and their scopes are managed with [`./publisher.sh creds`](../README.md#credentials) or the `/v1/creds` endpoints. - -* **Web session:** `POST /v1/auth/login`, then send the cookie with every request (`credentials: "include"` in `fetch`). Writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from this API's origin or `AUTH_WEB_ORIGINS`. Sessions end after 30 minutes idle or 12 hours. -* **HTTP Basic:** username and password on every request, e.g. `curl -u analyst: .../v1/stats/publications`. - -> [!WARNING] -> Basic auth sends the password with every request. Use it over HTTPS only. - -Scope changes apply on the credential's next request. Each endpoint's description names the scopes it needs. - -## Managing Credentials - -A credential can only grant scopes it holds, can't change itself, and can't change a credential holding scopes it lacks. These return `403`. - -`GET /v1/creds/{username}` returns an opaque `ETag`. Send it back unchanged as `If-Match` to change, reset or delete that credential: a missing header gets `428`, a stale one `412`. Responses that change a credential return its new `ETag`. Generated passwords appear only in the response that creates them. - -## Managing Platform Adapters - -`/v1/platforms/adapters` lists every installed adapter, with its source, version tag, commit and who last changed it, and takes the same `ETag` and `If-Match` rules as credentials. A disabled adapter disappears from `/v1/platforms` and can't publish or link accounts, but still revokes tokens so users can unlink. Uninstalling is refused with `409` while accounts are linked through the adapter; disable it instead, or remove it with `./publisher.sh platforms remove --force`. - -Installing (`POST /v1/platforms/adapters`) and updating (`POST /v1/platforms/adapters/{id}/update`) run code from the repository on the server, so they need an administrator and a GitHub repository of an org in `PLATFORMS_GITHUB_ORGS`. Both take an optional `tag`, defaulting to the newest version tag, and return `202` with the job's URL in `Location`. Poll the job until its `state` is `succeeded` or `failed`; its `log` has the end of the git and pip output. An adapter has one job at a time; another gets `409`. - -An adapter's settings are at `/v1/platforms/adapters/{id}/config`, for administrators only. `GET` lists each setting, with secrets only marked `is_set`, and the inputs its setup takes. `PATCH` changes the settings sent and keeps the rest; `POST .../setup` runs the adapter's setup and replaces them all, with a file input sent as the file's text. `configured` in the adapter list says whether every required setting has a value. - -## Managing Gateway Clients - -`/v1/gateway-clients/registry` lists every gateway client, with who last changed it, and takes the same `ETag` and `If-Match` rules as credentials (scopes `gc:read` and `gc:write`). `GET /v1/gateway-clients/registry/suggest?msisdn=` returns the details and PLMN candidates a create would use, to prefill a form for the administrator to confirm; see [how suggest works](gateway-clients.md#how-suggest-works). A disabled client disappears from the public `/v1/gateway-clients`. - -## Pagination - -List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. Follow `next` and `prev` as they are: they keep your filters and `limit`, and are `null` on the last and first page. Don't build the `cursor` yourself. - -`since` and `until` take ISO-8601 times, UTC if there's no offset. `since` is inclusive and `until` exclusive. - -## Audit Events - -`GET /v1/audit-events` needs `audit:read`, and each area also needs its read scope: `creds:read` for `auth.*` and `creds.*` events, `platforms:read` for `platforms.*`, `gc:read` for `gateway_clients.*`. Events from areas you can't read are left out. - -| Action | Recorded when | -| :--- | :--- | -| `creds.create`, `creds.update`, `creds.reset_password`, `creds.revoke_sessions`, `creds.delete` | A credential changes, over the API or the CLI. `details` says what changed, never secrets. | -| `auth.login` | A web session starts, or a login for an existing username is refused | -| `auth.logout` | A web session ends | -| `platforms.add`, `platforms.update`, `platforms.remove` | An adapter is installed, updated or removed. `details` has the source URL and commits. | -| `platforms.enable`, `platforms.disable` | An adapter is offered to users again, or hidden from them | -| `platforms.configure`, `platforms.setup` | An adapter's settings are changed, or set up from inputs. `details` has the keys saved. | -| `gateway_clients.create`, `gateway_clients.update`, `gateway_clients.delete` | A gateway client changes. `details` has its fields, or what changed. | -| `gateway_clients.enable`, `gateway_clients.disable` | A gateway client is listed publicly again, or hidden | - -`outcome` is `success`, `denied` (the change went beyond the actor's scopes) or `failed` (a refused login for an existing username: wrong password or disabled). Successful HTTP Basic requests aren't recorded, since every request authenticates. `actor` is the username at the time, and null for the CLI or a failed login. Events are kept for `AUDIT_RETENTION_DAYS` (365 by default). - -## Publishing - -`POST /v1/publications` takes JSON and tags payloads with protocol `https`. See [Offline Publishing](../README.md#offline-publishing) for how the protocol and `tag` decide whether offline payloads publish. - -It only checks the payload's structure before queueing it. Decryption, payload type and adapter errors happen later, so they're logged on the server, not returned. - -## Errors - -Error bodies are `{"error": ""}`. The message is meant to be shown to users; the server logs more detail. - -| Status | Meaning | -| :--- | :--- | -| `400 Bad Request` | Invalid parameters or payload, invalid cursor, or invalid time window | -| `401 Unauthorized` | Missing or invalid credentials | -| `403 Forbidden` | Origin not allowed, missing scope, or a credential change beyond your scopes | -| `404 Not Found` | The resource doesn't exist | -| `409 Conflict` | Already exists, in use, or changed by another request at the same moment; retry | -| `412 Precondition Failed` | `If-Match` doesn't match the current `ETag`; reload and retry | -| `422 Unprocessable Entity` | Validation error | -| `428 Precondition Required` | `If-Match` header missing | -| `429 Too Many Requests` | Rate limited (login and Basic auth) | -| `500 Internal Server Error` | Unexpected server error | diff --git a/docs/smtp.md b/docs/smtp.md deleted file mode 100644 index 98dc2c7..0000000 --- a/docs/smtp.md +++ /dev/null @@ -1,66 +0,0 @@ -# SMTP Transport - -Publishes RelaySMS payloads received by email. Incoming mails are picked up by polling a mailbox over IMAP (`publisher/smtp/listener.py`), authenticated, and queued for publication. - -## Message Format - -The body of each email must be a JSON object: - -```json -{ - "address": "5447c1f50558292bd9df723f9fdc0b06b892199c7dcfaad5164f2d94dfd3470a", - "dialing_code": "237", - "text": "" -} -``` - -| Field | Type | Description | -| :--- | :--- | :--- | -| address | string | Sender ID from the gateway client, or an E.164 phone number | -| dialing_code | string | Optional. Sender's country calling code, for publication stats | -| text | string | Base64-encoded serialized payload | - -## How It Works - -1. Waits for unseen mail in `SMTP_IMAP_MAIL_FOLDER` with IMAP IDLE. List several folders, such as `INBOX,Spam`, to catch mail a provider misfiled. -2. Rejects each email unless the sender is allowlisted and authenticated (see [Security](#security)). -3. Parses the body as JSON and validates the payload. -4. Queues the payload for publication. -5. Deletes the handled emails, in one call per folder. - -## Security - -Two checks must both pass before a message is queued: - -**Allowlist:** `SMTP_ALLOWED_SENDERS` lists exact addresses and domains, comma-separated. Empty allows nothing. - -> [!WARNING] -> A domain entry trusts every authenticated sender on that domain, since DKIM vouches for the domain, not the mailbox. List an exact address when only one sender should be trusted. - -**Authentication:** the listener can't re-check SPF, which needs the sender's IP. It reads the `Authentication-Results` header ([RFC 8601](https://www.rfc-editor.org/rfc/rfc8601)) that the mailbox's own server added, and only from the `authserv-id` in `SMTP_TRUSTED_AUTHSERV_ID`, since a sender can forge such a header from anyone else. `SMTP_REQUIRE_DKIM` and `SMTP_REQUIRE_SPF` choose the verdicts required. `SMTP_VERIFY_DKIM_INDEPENDENTLY=true` also re-verifies the DKIM signature against DNS. - -## Configuration - -| Variable | Default | Description | -| :--- | :--- | :--- | -| `SMTP_TRANSPORT_ENABLED` | `false` | Enables the listener. | -| `SMTP_IMAP_SERVER` | - | IMAP host. Required when enabled. | -| `SMTP_IMAP_PORT` | `993` | IMAP port. | -| `SMTP_IMAP_USERNAME` | - | Required when enabled. | -| `SMTP_IMAP_PASSWORD` | - | Required when enabled. | -| `SMTP_IMAP_MAIL_FOLDER` | `INBOX` | Comma-separated folders to poll. | -| `SMTP_TLS_CLIENT_CERTIFICATE` / `SMTP_TLS_CLIENT_KEY` | - | Optional mTLS client cert, only if the provider requires one. | -| `SMTP_ALLOWED_SENDERS` | - | Comma-separated allowlist of addresses and/or domains. | -| `SMTP_TRUSTED_AUTHSERV_ID` | - | `authserv-id` whose `Authentication-Results` verdicts are trusted. | -| `SMTP_REQUIRE_DKIM` | `true` | Require `dkim=pass`. | -| `SMTP_REQUIRE_SPF` | `true` | Require `spf=pass`. | -| `SMTP_VERIFY_DKIM_INDEPENDENTLY` | `false` | Also re-verify the DKIM signature via DNS. | -| `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` | - | Comma-separated allowlist of protocols allowed to publish offline payloads. Messages queued by this listener are tagged `smtp`; if set and `smtp` isn't listed, offline payloads from email are discarded. | - -## Running - -```sh -python3 -m publisher.smtp -``` - -`scripts/run.sh` and the `relaysms-publisher-smtp` unit start it too; it exits at once while `SMTP_TRANSPORT_ENABLED` is off. diff --git a/docs/theme/custom.css b/docs/theme/custom.css new file mode 100644 index 0000000..ad14727 --- /dev/null +++ b/docs/theme/custom.css @@ -0,0 +1,4 @@ +/* Tables fill the column and keep long code readable. */ +.content table { width: 100%; } +.content td { vertical-align: top; } +.content td code { overflow-wrap: anywhere; } diff --git a/install.sh b/install.sh index ee664b5..22c8a81 100755 --- a/install.sh +++ b/install.sh @@ -52,6 +52,7 @@ BRANCH="${BRANCH:-main}" CARGO_BIN="$HOME/.cargo/bin" DEPS_MARKER="/var/lib/relaysms-publisher-deps-installed" SITE_NAME="${SITE_NAME:-}" +DOCS_SITE_NAME="${DOCS_SITE_NAME:-}" LETSENCRYPT_EMAIL="${LETSENCRYPT_EMAIL:-}" SKIP_NGINX="${SKIP_NGINX:-0}" FORCE_DEPS="${FORCE_DEPS:-0}" @@ -101,7 +102,8 @@ Usage: install.sh [OPTIONS] --branch BRANCH Git branch to install (default: main) --site-name DOMAIN Domain to front with nginx + Let's Encrypt --letsencrypt-email EMAIL Email for Let's Encrypt renewal notices - --skip-nginx Skip the nginx/TLS setup entirely + --docs-site-name DOMAIN Domain to serve the manual on with nginx + Let's Encrypt + --skip-nginx Skip the nginx/TLS setup entirely, the manual's site included --force-deps Reinstall system dependencies even if already marked done --setup-db {mysql|postgres} Install and provision a database server, or connect to an existing one --db-existing Use an already-running database server instead of installing one locally @@ -168,6 +170,14 @@ parse_args() { SITE_NAME="${1#*=}" shift ;; + --docs-site-name) + DOCS_SITE_NAME="$2" + shift 2 + ;; + --docs-site-name=*) + DOCS_SITE_NAME="${1#*=}" + shift + ;; --letsencrypt-email) LETSENCRYPT_EMAIL="$2" shift 2 @@ -577,14 +587,44 @@ configure_nginx() { } fi validate_hostname "--site-name" "$site" + SITE_NAME="$site" install_nginx_certbot + install_site "$site" render_nginx_site "$interactive" +} + +configure_docs_site() { + if [ "${SKIP_NGINX:-0}" = "1" ]; then + return + fi + + local site="$DOCS_SITE_NAME" interactive=0 + if [ -z "$site" ]; then + # Without --docs-site-name, ask only when the API has a site, since the manual's site needs nginx and certbot. + [ -n "$SITE_NAME" ] || return 0 + interactive=1 + prompt site "Domain name to serve the manual on (e.g. docs.publisher.example.com, blank to skip): " "" + [ -n "$site" ] || { + log "Skipping the manual's site" + return + } + fi + validate_hostname "--docs-site-name" "$site" + + log "Building the manual" + "$INSTALL_DIR/scripts/build-book.sh" + install_nginx_certbot + install_site "$site" render_docs_site "$interactive" +} +# $1=domain $2=function that writes the site file, called as "$2 DOMAIN FILE" $3=1 to ask before certbot and for its email +install_site() { + local site="$1" render="$2" interactive="$3" local conf_dest="/etc/nginx/sites-available/${site}.conf" if [ -f "$conf_dest" ]; then log "nginx site $conf_dest already exists, leaving it untouched" else - render_nginx_site "$site" "$conf_dest" + "$render" "$site" "$conf_dest" log "Wrote $conf_dest" fi ln -sf "$conf_dest" "/etc/nginx/sites-enabled/${site}.conf" @@ -607,12 +647,11 @@ configure_nginx() { ;; esac - local email="${LETSENCRYPT_EMAIL:-}" - if [ -z "$email" ] && [ "$interactive" = "1" ]; then - prompt email "Email for Let's Encrypt renewal notices (optional): " "" + if [ -z "$LETSENCRYPT_EMAIL" ] && [ "$interactive" = "1" ]; then + prompt LETSENCRYPT_EMAIL "Email for Let's Encrypt renewal notices (optional): " "" fi - request_certificate "$site" "$email" || error "certbot failed to obtain a certificate for $site" + request_certificate "$site" "$LETSENCRYPT_EMAIL" || error "certbot failed to obtain a certificate for $site" log "Certificate installed for $site" enable_nginx_http2 "$conf_dest" @@ -817,6 +856,7 @@ main() { run_migrations install_services configure_nginx + configure_docs_site configure_observability log "Installation complete" diff --git a/manage.sh b/manage.sh index e69b8bf..f4e0a95 100755 --- a/manage.sh +++ b/manage.sh @@ -252,6 +252,10 @@ cmd_update() { systemctl restart "$svc" done systemctl start "$TARGET_UNIT" + # book/ exists only when install.sh set up a site for the manual, so other installs skip the rebuild. + if [ -d book ]; then + ./scripts/build-book.sh + fi [ "$config_ok" = "1" ] || error "Update applied, but .env has errors (above); services using those settings won't start" log "Update complete" } @@ -267,6 +271,17 @@ restored on failure. EOF } +docs_usage() { + cat <<'EOF' +Usage: manage.sh docs [DOMAIN] + +Builds the manual into book/, then renders its nginx site from the template, +reattaches or obtains its certificate and reloads nginx. DOMAIN defaults to +this install's manual site. The previous file is kept as .conf.bak and +restored on failure. manage.sh update rebuilds the manual afterwards. +EOF +} + # Matches publisher_rest_ (this install) or an unsuffixed publisher_rest. detect_nginx_site() { local port sites=() @@ -278,9 +293,18 @@ detect_nginx_site() { basename "${sites[0]}" .conf } +detect_docs_site() { + local sites=() + mapfile -t sites < <(grep -lF "root $INSTALL_DIR/book;" /etc/nginx/sites-available/*.conf 2>/dev/null) + [ "${#sites[@]}" -eq 1 ] || + error "Found ${#sites[@]} matching nginx sites; pass the domain: $0 docs DOMAIN" + basename "${sites[0]}" .conf +} + +# $1=domain $2=site file $3=function that writes it, called as "$3 DOMAIN FILE" install_nginx_site() { - local site="$1" conf="$2" - render_nginx_site "$site" "$conf" || return 1 + local site="$1" conf="$2" render="$3" + "$render" "$site" "$conf" || return 1 ln -sf "$conf" "/etc/nginx/sites-enabled/${site}.conf" || return 1 # Re-adds the 443 block that re-rendering dropped. @@ -294,26 +318,21 @@ install_nginx_site() { nginx -t || return 1 } -cmd_nginx() { - case "${1:-}" in - -h | --help) - nginx_usage - return - ;; - esac - require_root +require_nginx() { if ! command -v nginx &>/dev/null || ! command -v certbot &>/dev/null; then error "nginx and certbot must be installed" fi +} - local site="${1:-}" - [ -n "$site" ] || site=$(detect_nginx_site) +# $1=domain $2=function that writes its site file +apply_nginx_site() { + local site="$1" render="$2" validate_hostname "DOMAIN" "$site" local conf="/etc/nginx/sites-available/${site}.conf" [ -f "$conf" ] && cp -p "$conf" "$conf.bak" - if ! install_nginx_site "$site" "$conf"; then + if ! install_nginx_site "$site" "$conf" "$render"; then if [ -f "$conf.bak" ]; then cp -p "$conf.bak" "$conf" nginx -t && systemctl reload nginx @@ -324,6 +343,36 @@ cmd_nginx() { log "nginx site $site updated" } +cmd_nginx() { + case "${1:-}" in + -h | --help) + nginx_usage + return + ;; + esac + require_root + require_nginx + local site="${1:-}" + [ -n "$site" ] || site=$(detect_nginx_site) + apply_nginx_site "$site" render_nginx_site +} + +cmd_docs() { + case "${1:-}" in + -h | --help) + docs_usage + return + ;; + esac + require_root + require_nginx + local site="${1:-}" + [ -n "$site" ] || site=$(detect_docs_site) + validate_hostname "DOMAIN" "$site" + "$INSTALL_DIR/scripts/build-book.sh" + apply_nginx_site "$site" render_docs_site +} + cmd_uninstall() { require_root local confirm @@ -353,7 +402,7 @@ cmd_uninstall() { } usage() { - echo "Usage: $0 {start|stop|restart|status|logs|enable|disable|check|migrate|update|nginx|uninstall}" + echo "Usage: $0 {start|stop|restart|status|logs|enable|disable|check|migrate|update|nginx|docs|uninstall}" exit 1 } @@ -375,6 +424,10 @@ main() { shift cmd_nginx "$@" ;; + docs) + shift + cmd_docs "$@" + ;; update) shift cmd_update "$@" diff --git a/observability/README.md b/observability/README.md deleted file mode 100644 index ae5b2ab..0000000 --- a/observability/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# Observability - -Self-hosted tracing, metrics, log export (SigNoz) and uptime monitoring -(Uptime Kuma) for Publisher's 5 services. Optional: -`OTEL_EXPORTER_OTLP_ENDPOINT` is blank by default. - -## Setup - -```bash -sudo ./install.sh --setup-observability -# or, against an already-installed Publisher: -sudo ./scripts/setup-observability.sh -``` - -Flags (full list: `scripts/setup-observability.sh --help`): - -| Flag | Does | -| --- | --- | -| `--observability-site-name` (`--site-name`) | Fronts SigNoz with nginx + TLS | -| `--observability-kuma-site-name` (`--kuma-site-name`) | Fronts Kuma with nginx + TLS, different domain from the above | -| `--signoz-email` / `--signoz-password` | SigNoz admin account | -| `--signoz-smtp-host/-port/-username/-password/-from` | SMTP relay for SigNoz's email alert channel | - -Prerequisites: Docker Engine + Compose plugin. - -## SigNoz - -**Automated**: generates `observability/signoz/casting.yaml` from -`casting.yaml.template` (Postgres credentials, ports, SMTP), casts it, -binds the UI/OTLP ports to `127.0.0.1`, fixes the opamp -misconfiguration that keeps the OTLP receiver closed, creates the admin -account. - -**Manual**: - -```bash -curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_INSTALL_DIR=/usr/local/bin bash - -sed \ - -e "s/__POSTGRES_DB__/signoz/" -e "s/__POSTGRES_USER__/signoz/" \ - -e "s/__POSTGRES_PASSWORD__/$(openssl rand -hex 24)/" \ - -e "s/__SIGNOZ_PORT__/8090/" -e "s/__OTLP_GRPC_PORT__/4317/" -e "s/__OTLP_HTTP_PORT__/4318/" \ - -e "s/__SMTP_SMARTHOST__//" -e "s/__SMTP_USERNAME__//" -e "s/__SMTP_PASSWORD__//" -e "s/__SMTP_FROM__//" \ - -e "s|__SIGNOZ_EXTERNAL_URL__|http://localhost:8090|" \ - observability/signoz/casting.yaml.template > observability/signoz/casting.yaml - -foundryctl cast -f observability/signoz/casting.yaml -docker compose \ - -f pours/deployment/compose.yaml \ - -f observability/signoz/docker-compose.override.yml \ - up -d - -until curl -fs http://127.0.0.1:8090/api/v1/version | grep -q setupCompleted; do sleep 2; done -curl -X POST http://127.0.0.1:8090/api/v1/register \ - -H "Content-Type: application/json" \ - -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@relaysms.local","password":"<12+ chars, upper, lower, number, symbol>"}' -``` - -`metastore.kind: postgres` is the default in the template; set it to -`sqlite` for a single-node setup. Retention isn't a `casting.yaml` field: -set it in the UI, under Settings > Workspace > Retention Controls. - -> [!IMPORTANT] -> `casting.yaml` is generated only once. To change a running install, edit -> `observability/signoz/casting.yaml`, then re-run -> `foundryctl cast -f observability/signoz/casting.yaml`. Only the `signoz` -> container is recreated; Postgres and ClickHouse are untouched. - -## Uptime Kuma - -**Automated**: brings up the container, bound to `127.0.0.1:3001` unless -another port is chosen, which it saves to `observability/uptime-kuma/.env`. - -**Manual**: `docker compose -f observability/uptime-kuma/docker-compose.yml up -d` - -Reach it through its own reverse-proxy domain (below) or -`ssh -L 3001:127.0.0.1:3001 @`, then -. Create the admin account on first visit, then add -monitors: - -- **REST**: Monitor Type "HTTP(s)", URL `https:///health`. -- **gRPC**: Monitor Type "GRPC(S) - Keyword". Fields: - - gRPC URL: `:443` - - Enable TLS: on - - Service Name: `grpc.health.v1.Health` - - Method: `check` - - gRPC Body: `{"service":"publisher.v3.Publisher"}` - - Keyword: `SERVING` - - Protobuf: - - ```proto - syntax = "proto3"; - package grpc.health.v1; - - message HealthCheckRequest { - string service = 1; - } - - message HealthCheckResponse { - enum ServingStatus { - UNKNOWN = 0; - SERVING = 1; - NOT_SERVING = 2; - SERVICE_UNKNOWN = 3; - } - ServingStatus status = 1; - } - - service Health { - rpc Check(HealthCheckRequest) returns (HealthCheckResponse); - } - ``` - -- **Worker**: Monitor Type "Push", interval just above 60s. Wire the push - URL Kuma shows into `UPTIME_KUMA_WORKER_PUSH_URL` and - `./manage.sh restart`. -- **SMTP listener**: same as Worker, interval just above 20s, into - `UPTIME_KUMA_SMTP_PUSH_URL`. -- **Beat**: Push monitor, optional. - -## Reverse proxy - -**Automated**: `--site-name`/`--kuma-site-name` on `setup-observability.sh` -write the site files, symlink them, run `certbot`. - -**Manual**: - -```bash -sed -e "s/__SERVER_NAME__//g" \ - observability/nginx-observability.conf.template \ - > /etc/nginx/sites-available/.conf -ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ - -sed -e "s/__SERVER_NAME__//g" \ - observability/nginx-uptime-kuma.conf.template \ - > /etc/nginx/sites-available/.conf -ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ - -nginx -t && systemctl reload nginx -certbot --nginx -d -certbot --nginx -d -``` - -## Turning tracing on/off per service - -> [!NOTE] -> systemd installs don't include `requirements-observability.txt` until you -> install it: - -```bash -cd /opt/relaysms/relaysms-publisher -sudo venv/bin/pip install --quiet -r requirements-observability.txt -``` - -```bash -sudo sed -i 's|^OTEL_EXPORTER_OTLP_ENDPOINT=.*|OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317|' /opt/relaysms/relaysms-publisher/.env -./manage.sh restart -``` - -Blank the value and restart to turn it back off. - -## Alerts - -SigNoz UI, Alerts > New Alert > Logs signal, paste the filter below into the -search bar, set the threshold to count > 0 over a rolling window, attach a -notification channel. `CONTAINS` is a real query-builder operator; wrap the -`OR` group in parentheses since `AND` binds tighter than `OR`. - -- **Publish failures** (`tasks/publication_task.py`): - - ``` - service.name = 'relaysms-publisher-worker' AND (body CONTAINS 'Failed to process payload' OR body CONTAINS 'Failed to publish message' OR body CONTAINS 'An unexpected error occurred during task processing') - ``` - -- **Platform adapter failures** (`publisher/platforms/ipc.py`): - - ``` - service.name = 'relaysms-publisher-worker' AND (body CONTAINS 'Subprocess failed' OR body CONTAINS 'Malformed JSON response' OR body CONTAINS 'Subprocess execution timed out' OR body CONTAINS 'Unexpected failure during IPC invocation') - ``` - -- **Key management failures** (`publisher/keys.py`): - - ``` - body CONTAINS 'Failed to generate server identity keys' OR body CONTAINS 'Token hash missing' - ``` diff --git a/observability/nginx-observability.conf.template b/observability/nginx-observability.conf.template index 9ea4a30..d044aeb 100644 --- a/observability/nginx-observability.conf.template +++ b/observability/nginx-observability.conf.template @@ -1,5 +1,5 @@ # nginx reverse proxy site template for SigNoz. -# Setup: observability/README.md +# Setup: docs/operating/observability.md upstream signoz_ui { server 127.0.0.1:8080; diff --git a/observability/nginx-uptime-kuma.conf.template b/observability/nginx-uptime-kuma.conf.template index 6d1aaba..50e182b 100644 --- a/observability/nginx-uptime-kuma.conf.template +++ b/observability/nginx-uptime-kuma.conf.template @@ -1,5 +1,5 @@ # nginx reverse proxy site template for Uptime Kuma. -# Setup: observability/README.md +# Setup: docs/operating/observability.md upstream uptime_kuma_ui { server 127.0.0.1:3001; diff --git a/observability/signoz/casting.yaml.template b/observability/signoz/casting.yaml.template index 56b305f..4bbd2f0 100644 --- a/observability/signoz/casting.yaml.template +++ b/observability/signoz/casting.yaml.template @@ -3,7 +3,7 @@ # Not run directly: the placeholders below are filled in by # scripts/setup-observability.sh, which writes # observability/signoz/casting.yaml (gitignored, contains real -# credentials). See observability/README.md for the manual equivalent. +# credentials). See docs/operating/observability.md for the manual equivalent. # # Left unset, Foundry defaults the Postgres fields to the literal string # "signoz", so don't cast this template file directly. diff --git a/publisher.sh b/publisher.sh index a346fb8..8378566 100755 --- a/publisher.sh +++ b/publisher.sh @@ -17,7 +17,7 @@ ENV_FILE="$INSTALL_DIR/.env" VENV_DIR="$INSTALL_DIR/venv" [ -x "$VENV_DIR/bin/python3" ] || - error "Virtualenv not found at $VENV_DIR. Run install.sh, or see Development in README.md." + error "Virtualenv not found at $VENV_DIR. Run install.sh, or see Development setup in CONTRIBUTING.md." INSTANCE_NAME="$(read_instance_name)" SERVICE_USER="$(detect_service_user)" diff --git a/publisher/api/grpc/server.py b/publisher/api/grpc/server.py index 94a67c8..cdecb53 100644 --- a/publisher/api/grpc/server.py +++ b/publisher/api/grpc/server.py @@ -78,7 +78,7 @@ def _bind_port(grpc_server: grpc.Server) -> None: return cert_file, key_file = grpc_config.tls_cert_file, grpc_config.tls_key_file - assert cert_file and key_file # config requires both when TLS is on + assert cert_file and key_file # config refuses TLS without both files try: credentials = _load_ssl_credentials(Path(cert_file), Path(key_file)) except FileNotFoundError as e: @@ -105,7 +105,8 @@ def serve() -> None: grpc_server = _build_server(grpc_config.max_workers) _bind_port(grpc_server) - # Stopping inside the handler hangs if a second signal arrives mid-stop. + # The handler only sets this flag: stopping inside it would hang if a second + # signal arrived while the server was stopping. stop_requested = threading.Event() def request_stop(signum: int, _frame) -> None: diff --git a/publisher/api/grpc/utils.py b/publisher/api/grpc/utils.py index 8ab9354..1b27d21 100644 --- a/publisher/api/grpc/utils.py +++ b/publisher/api/grpc/utils.py @@ -55,7 +55,6 @@ def validate_client_ephemeral_public_keys(keys) -> None: def find_adapter(platform: str, proto_id: int) -> PlatformAdapter: - # Its own session, so none stays open while the adapter runs. with get_session() as s: return platform_adapters.get_for_protocol(s, platform, proto_id) @@ -63,7 +62,8 @@ def find_adapter(platform: str, proto_id: int) -> PlatformAdapter: def call_adapter(adapter: PlatformAdapter, method: str, request: Any) -> dict: """Call an adapter, turning its errors into gRPC errors. - Rejected input and codes reach the client; anything else is logged. + Invalid input, failed authentication and rate limits reach the client with the + adapter's message; any other error is logged and the client gets a generic one. """ try: return ipc.call(adapter, method, request) diff --git a/publisher/api/rest/app.py b/publisher/api/rest/app.py index 534e014..3b6e833 100644 --- a/publisher/api/rest/app.py +++ b/publisher/api/rest/app.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -# Query and path values are already in the URL; body values can be secrets. +# Validation errors echo values from these locations only. ECHOED_INPUT_LOCATIONS = frozenset({"query", "path"}) MAX_ECHOED_INPUT_LENGTH = 50 diff --git a/publisher/api/rest/v1/audit.py b/publisher/api/rest/v1/audit.py index e80a2f1..5856ca7 100644 --- a/publisher/api/rest/v1/audit.py +++ b/publisher/api/rest/v1/audit.py @@ -46,7 +46,7 @@ def list_audit_events( ), db: Session = Depends(get_db), ) -> AuditEventPage: - """Newest first. Scope: audit:read, plus creds:read or platforms:read per area.""" + """Newest first. Scope: audit:read, plus each area's read scope.""" since, until = check_time_range(since, until) page = audit_event.list_events( db, diff --git a/publisher/api/rest/v1/auth.py b/publisher/api/rest/v1/auth.py index 04e593f..4abab83 100644 --- a/publisher/api/rest/v1/auth.py +++ b/publisher/api/rest/v1/auth.py @@ -105,7 +105,8 @@ def _request_origin(request: Request) -> str | None: def _record_failed_login(db: Session, username: str, method: str) -> None: - # Not for unknown usernames: anyone can send those, so they'd flood the table. + # Recorded only for existing usernames: anyone can send unknown ones, which + # would fill the audit table. target = get_by_username(db, username) if target is None: return diff --git a/publisher/api/rest/v1/creds.py b/publisher/api/rest/v1/creds.py index ce8de79..0eaecaf 100644 --- a/publisher/api/rest/v1/creds.py +++ b/publisher/api/rest/v1/creds.py @@ -72,7 +72,7 @@ def _model_errors( log=f"{by}: {e}", ) from None except CredentialPermissionError as e: - # Keep the denial, not the change. + # Undo the refused change, then record and commit the denial. db.rollback() audit_event.record( db, diff --git a/publisher/api/rest/v1/params.py b/publisher/api/rest/v1/params.py index f02cc0d..be7b307 100644 --- a/publisher/api/rest/v1/params.py +++ b/publisher/api/rest/v1/params.py @@ -44,7 +44,8 @@ def check_if_match(request: Request, current: str, what: str) -> None: if_match = request.headers.get("If-Match") if if_match is None: raise ApiError(428, "If-Match header required.") - # nginx gzip turns our ETag weak (W/"..."), and ours are version-based anyway. + # nginx gzip marks the ETag weak (W/"..."). Ours come from the row version, not + # the body, so a weak one still matches. if if_match.strip().removeprefix("W/") != current: raise ApiError( 412, diff --git a/publisher/api/rest/v1/platform_adapters.py b/publisher/api/rest/v1/platform_adapters.py index b7c3d6a..8892bdd 100644 --- a/publisher/api/rest/v1/platform_adapters.py +++ b/publisher/api/rest/v1/platform_adapters.py @@ -105,7 +105,6 @@ def _adapter_errors(context: AuthContext): except (AdapterInUseError, AdapterConflictError) as e: raise ApiError(409, str(e), log=f"{by}: {e}") from None except AdapterFaultError as e: - # Its message can name files on this server. message = "The adapter or its files are broken; see the server log." raise ApiError(502, message, log=f"{by}: {e}") from None except AdapterError as e: @@ -292,7 +291,8 @@ def setup_config( _require_administrator(context, "Adapter settings") adapter = _load(db, adapter_id) check_if_match(request, _etag(adapter), "adapter") - # End the read before the adapter runs; the version still catches a change. + # End the read transaction before the adapter runs. A change made meanwhile + # still fails the save, which checks the adapter's version. db.commit() with _adapter_errors(context): adapter_config.setup(db, adapter, body.inputs, actor=context.credential) @@ -320,7 +320,8 @@ def _queue( try: run_adapter_job.delay(str(job.id)) except Exception as e: - # Fail it now, or its lock would block the adapter until cleanup. + # Fail the job now. Otherwise it stays active and blocks every other job for + # the adapter until cleanup fails it as abandoned. jobs.finish(db, job.id, state="failed", log=[f"Queueing failed: {e}"]) db.commit() raise ApiError( diff --git a/publisher/api/rest/v1/platforms.py b/publisher/api/rest/v1/platforms.py index 12fe02e..fa4eca1 100644 --- a/publisher/api/rest/v1/platforms.py +++ b/publisher/api/rest/v1/platforms.py @@ -88,8 +88,6 @@ def get_platform_oauth_client_metadata( detail="OAuth client metadata file not found for this platform", ) from exc except ValueError as exc: - # The error names the file's path on this server, so the client only gets - # a generic message. logger.error("Invalid OAuth client metadata: %s", exc) raise HTTPException( status_code=500, detail="OAuth client metadata is invalid" diff --git a/publisher/api/rest/v1/publications.py b/publisher/api/rest/v1/publications.py index c374304..52f74da 100644 --- a/publisher/api/rest/v1/publications.py +++ b/publisher/api/rest/v1/publications.py @@ -26,7 +26,7 @@ def create_publications(body: PublishRestContentRequest) -> PublishContentRespon """Queues an SMS payload for publishing. Multi-part payloads are kept until every segment arrives, then published. Payloads - are tagged with protocol `https`; see Offline Publishing in the README for + are tagged with protocol `https`; see Offline publishing in the manual for `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors happen later in the queue, so they're logged, not returned. """ diff --git a/publisher/api/rest/v1/schemas.py b/publisher/api/rest/v1/schemas.py index ca3ce23..e11dc8a 100644 --- a/publisher/api/rest/v1/schemas.py +++ b/publisher/api/rest/v1/schemas.py @@ -42,7 +42,8 @@ class OAuthClientMetadata(BaseModel): client_name: str client_uri: str scope: str - # The same for every atproto web client, so settings don't include them. + # The same for every atproto web client, so the adapter's settings leave them + # out and these defaults fill them in. dpop_bound_access_tokens: bool = True application_type: str = "web" grant_types: list[str] = ["authorization_code", "refresh_token"] diff --git a/publisher/cli/seed.py b/publisher/cli/seed.py index feac9ec..237af43 100644 --- a/publisher/cli/seed.py +++ b/publisher/cli/seed.py @@ -173,7 +173,8 @@ def gateway_clients_command(count): while added < count: msisdn = _random_mobile(random.choice(COUNTRIES)) suggestion = gateway_clients.suggest(msisdn) - # Picks for the ambiguous ones, as an administrator would. + # Takes the first candidate when the operator is ambiguous, as an + # administrator would. candidate = suggestion.candidates[0] try: client = gateway_clients.create( diff --git a/publisher/credentials.py b/publisher/credentials.py index b504174..082484d 100644 --- a/publisher/credentials.py +++ b/publisher/credentials.py @@ -33,7 +33,8 @@ password_hasher = PasswordHasher() -# Each scope here can only be granted with the scope it maps to. +# Granting the first scope of each pair also needs the second, e.g. gc:write needs +# gc:read. _REQUIRES = { Scope.STATS_PUBLICATIONS_REASONS: Scope.STATS_PUBLICATIONS_READ, Scope.GC_WRITE: Scope.GC_READ, diff --git a/publisher/db/__init__.py b/publisher/db/__init__.py index d66bfe1..5960e63 100644 --- a/publisher/db/__init__.py +++ b/publisher/db/__init__.py @@ -161,7 +161,8 @@ def _create_engine() -> Engine: else: _ensure_sqlite_parent_dir(database.sqlite_path) if database.encryption_enabled: - assert database.encryption_key is not None # config requires it here + # config refuses SQLCipher without a key + assert database.encryption_key is not None logger.debug("Using SQLCipher3 encryption for SQLite") from sqlcipher3 import dbapi2 as sqlcipher3_dbapi diff --git a/publisher/gateway_clients/mcc_mnc.py b/publisher/gateway_clients/mcc_mnc.py index a9ce946..1c05aa2 100644 --- a/publisher/gateway_clients/mcc_mnc.py +++ b/publisher/gateway_clients/mcc_mnc.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-only """MCC/MNC (PLMN) lookup in the vendored mcc_mnc_table.json snapshot. -See docs/gateway-clients.md. +See docs/gateway-clients/manage.md. """ import json @@ -30,7 +30,8 @@ ) # Snapshot rows that aren't operators. _NOT_OPERATORS = frozenset({"", "failed calls", "fix line", "unknown"}) -# A word in this many of a region's networks only weakly tells them apart. +# A word in at least this many of a region's network names is common, so sharing +# it is only a weak match. _COMMON_IN_REGION = 3 @@ -85,7 +86,7 @@ def _score( ): best = max(best, 2) elif shared: - best = max(best, 1) # Only words many of the region's networks have. + best = max(best, 1) # Only common words are shared. return best diff --git a/publisher/models/credential_session.py b/publisher/models/credential_session.py index 7c5732b..9d6a2ee 100644 --- a/publisher/models/credential_session.py +++ b/publisher/models/credential_session.py @@ -106,7 +106,8 @@ def get_active(session: Session, raw_token: str) -> CredentialSession | None: return None credential = credential_session.credential - # None if a login raced a delete, since sqlite doesn't enforce the foreign key. + # None when the credential was deleted while this session was being created, + # since SQLite doesn't enforce the foreign key. if credential is None or not credential.is_active: return None if credential_session.session_version != credential.session_version: diff --git a/publisher/models/platform_adapter_job.py b/publisher/models/platform_adapter_job.py index 5fe08b5..d5625a2 100644 --- a/publisher/models/platform_adapter_job.py +++ b/publisher/models/platform_adapter_job.py @@ -35,8 +35,8 @@ class PlatformAdapterJob(Base): from_commit: Mapped[str | None] = mapped_column(String(40), default=None) to_commit: Mapped[str | None] = mapped_column(String(40), default=None) state: Mapped[str] = mapped_column(String(16), default="queued") - # The adapter id while the job is active, NULL after. Being unique, it allows - # one active job per adapter on every database. + # The adapter id while the job is queued or running, NULL once it finishes. + # Its unique index allows one active job per adapter. lock: Mapped[str | None] = mapped_column(String(36), default=None) log: Mapped[str] = mapped_column(Text, default="") requested_by: Mapped[uuid.UUID | None] = mapped_column(Uuid, default=None) @@ -159,7 +159,8 @@ def delete_older_than(session: Session, cutoff: datetime.datetime) -> int: result = cast( CursorResult, session.execute( - # By finish time, so a job failed just now as abandoned stays readable. + # By finished_at, so an old job that cleanup just failed as abandoned + # is kept for the full retention period. delete(PlatformAdapterJob).where(PlatformAdapterJob.finished_at < cutoff) ), ) diff --git a/publisher/platforms/adapter_config.py b/publisher/platforms/adapter_config.py index f00c224..de1f93c 100644 --- a/publisher/platforms/adapter_config.py +++ b/publisher/platforms/adapter_config.py @@ -163,13 +163,14 @@ def _save( actor: Credential | None, changed: dict[str, Any], ) -> None: - # Touching the row moves its version on, so a concurrent change conflicts. + # Updating the row bumps its version, so a concurrent change to the adapter + # fails with a conflict. adapter.updated_at = utc_now() adapter.updated_by = actor.id if actor else None flush(session, adapter) audit_event.record( session, action, actor=actor, target=adapter, details={"keys": sorted(changed)} ) - # Written before the commit, unlike manager.activate: the flush holds the row, - # so concurrent saves reach the file in order. + # Written before the commit, unlike manager.activate: the flush locks the row + # until the commit, so concurrent saves write the file one at a time. write_config(settings, directory=Path(adapter.config_path)) diff --git a/publisher/platforms/manager.py b/publisher/platforms/manager.py index 4785fd3..5ed8c27 100644 --- a/publisher/platforms/manager.py +++ b/publisher/platforms/manager.py @@ -244,7 +244,8 @@ def _register(session: Session, adapter: PlatformAdapter, manifest: Manifest) -> def activate(adapter: PlatformAdapter) -> None: """Move a built version into place. Call it after the install or update commits. - Run before the commit, a failed commit would leave the new files with the old row. + If it ran first and the commit failed, the new files would be in place while the + database still names the old version. """ # Renames are atomic, so an adapter call sees the old version or the new. for current in (Path(adapter.path), Path(adapter.venv_path)): @@ -311,7 +312,8 @@ def update( """Register tag, or the newest version tag, for an adapter.""" tag = tag or latest_tag(adapter.source_url) commit, manifest = _build(adapter, tag, log) - # A tag that now names other code was moved, which a release shouldn't do. + # Refused: the installed tag now points to other code, so it was moved after + # its release. if tag == adapter.tag and commit != adapter.commit: _discard_build(adapter) raise AdapterError( @@ -363,7 +365,8 @@ def remove( if adapter.id in ("", ".", "..") or "/" in adapter.id: raise AdapterError(f"Unsafe adapter id {adapter.id!r}") linked = tokens.count_for_platform(session, adapter.name, adapter.proto_id) - # Their tokens can only be revoked upstream through the adapter. + # Refused while accounts are linked: without the adapter, their tokens can't + # be revoked at the platform. if linked and not force: raise AdapterInUseError( f"{linked} linked account(s) use {adapter.name!r}. Disable it instead, " diff --git a/publisher/publications.py b/publisher/publications.py index 39013b2..721ec12 100644 --- a/publisher/publications.py +++ b/publisher/publications.py @@ -84,7 +84,8 @@ def __init__( ): super().__init__(message, platform_name=platform_name) self.code = code - # Refreshed before the send failed, so still the one to store. + # A token the adapter refreshed before the send failed; it replaces the + # stored one. self.token = token @@ -354,7 +355,7 @@ def send(delivery: Delivery) -> dict | None: def store_token(session: Session, delivery: Delivery, new_token: dict | None) -> None: - """Store a token the adapter refreshed, unless it was unlinked meanwhile.""" + """Store a token the adapter refreshed, if the account is still linked.""" token = session.get(Token, delivery.token_id) if delivery.token_id else None if token and new_token and new_token != token.token_data.get("token"): update_token_data(session, token, {**token.token_data, "token": new_token}) diff --git a/publisher/tasks/celery_app.py b/publisher/tasks/celery_app.py index f7260f4..063b167 100644 --- a/publisher/tasks/celery_app.py +++ b/publisher/tasks/celery_app.py @@ -80,7 +80,7 @@ def make_celery() -> Celery: }, } if celery.worker_heartbeat_url: - # Uptime Kuma push-monitor heartbeat, see observability/README.md + # Uptime Kuma push-monitor heartbeat, see docs/operating/observability.md beat_schedule["worker-heartbeat"] = { "task": "tasks.heartbeat_task.ping_worker_heartbeat", "schedule": 60.0, diff --git a/publisher/tasks/publication_task.py b/publisher/tasks/publication_task.py index 22d5957..2ff20d8 100644 --- a/publisher/tasks/publication_task.py +++ b/publisher/tasks/publication_task.py @@ -82,8 +82,9 @@ def publish_message( """Validate and publish a payload, then record the outcome.""" delivery = refreshed_token = None try: - # Commits only after a successful send, so a failure leaves the payload's - # keys usable, and a copy of it arriving meanwhile waits on their row lock. + # Commits only after a successful send. A failure rolls back, so the payload's + # keys can be used again, and a duplicate that arrives during the send waits + # on their row lock. with get_session() as db: payload_raw, raw_segment, payload_type = publications.validate(text_payload) delivery = publications.prepare( @@ -96,7 +97,7 @@ def publish_message( tag=tag, ) if delivery is None: - # Incomplete multi-segment session, awaiting more parts. + # Segments of a multi-part payload are still missing. return new_token = publications.send(delivery) publications.finish(db, delivery, new_token) diff --git a/publisher/tokens.py b/publisher/tokens.py index 750f31f..2c93e68 100644 --- a/publisher/tokens.py +++ b/publisher/tokens.py @@ -26,10 +26,10 @@ def account(token: Token) -> Account: def revocation(session: Session, token: Token) -> Revocation | None: - """Read what revoking a token takes, before the token is deleted. + """Return the adapter and request that revoke a token; call it before deleting it. - None when no adapter serves its platform. Run it with revoke_upstream once the - deletion commits. + None when no adapter serves its platform. Pass the result to revoke_upstream + after the deletion commits. """ try: # Disabled adapters still revoke, so users can always unlink. @@ -47,7 +47,7 @@ def revocation(session: Session, token: Token) -> Revocation | None: def revoke_upstream(revocation: Revocation | None) -> None: - """Revoke at the platform, outside any transaction; failures are only logged.""" + """Revoke a token at the platform once its deletion commits; failures are logged.""" if revocation is None: return adapter, request = revocation diff --git a/requirements-observability.txt b/requirements-observability.txt index 8d1c08b..27f8cca 100644 --- a/requirements-observability.txt +++ b/requirements-observability.txt @@ -1,6 +1,6 @@ # OpenTelemetry SDK, exporter and auto-instrumentation, needed only to export # traces, metrics and logs. Without them the app's opentelemetry-api calls do -# nothing. See observability/README.md. +# nothing. See docs/operating/observability.md. # # Includes requirements.txt, since both pull in OTel packages that must # resolve to the same versions. diff --git a/requirements.txt b/requirements.txt index db42364..32d784f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==7.2.3 celery[sqlalchemy,redis]==5.6.3 cryptography==50.0.2 dkimpy==1.1.8 -fastapi==0.143.3 +fastapi==0.149.0 GitPython==3.2.1 grpc-interceptor==0.15.4 grpcio==1.84.0 diff --git a/scripts/build-book.sh b/scripts/build-book.sh new file mode 100755 index 0000000..17cf288 --- /dev/null +++ b/scripts/build-book.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-3.0-only +# Builds the manual in docs/ into book/ with pinned mdbook and mdbook-mermaid +# releases, downloaded once into .mdbook/. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" + +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +TOOLS_DIR="$ROOT_DIR/.mdbook" +SERVE=false + +MDBOOK_VERSION=0.5.3 +MERMAID_VERSION=0.17.1 +declare -A SHA256=( + ["mdbook-x86_64"]=dfb86d8e20a3fed91bf549aa450b9d7f46a275b700292e050cdfd3171732e7fd + ["mdbook-aarch64"]=428c91cf73fec7494cbefe6e170e12508c9d45d9d096104b7c62d47ff43911c2 + ["mdbook-mermaid-x86_64"]=42aca57eea8521971cc595fe4bb853c5b8b7d9f7138a2ec9f14a0a72c210922d + ["mdbook-mermaid-aarch64"]=b1782ecaf775bfbe069dead6cdd8ac7cc739d5e62d138bd0e800efec6467cf3a +) + +usage() { + cat <<'EOF' +Usage: build-book.sh [OPTIONS] + + --serve Serve the manual on http://localhost:3000 and rebuild on changes + -h, --help Show this help and exit +EOF +} + +parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --serve) + SERVE=true + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + usage >&2 + error "Unknown option: $1" + ;; + esac + done +} + +# $1=tool $2=version $3=GitHub repo +fetch_tool() { + local tool="$1" version="$2" repo="$3" dir="$TOOLS_DIR/$1-$2" key tarball + [ -x "$dir/$tool" ] && return + key="$tool-$(uname -m)" + [ -n "${SHA256[$key]:-}" ] || error "No $tool release for $(uname -m)" + tarball="$(mktemp)" + curl -fsSL -o "$tarball" \ + "https://github.com/$repo/releases/download/v$version/$tool-v$version-$(uname -m)-unknown-linux-musl.tar.gz" + echo "${SHA256[$key]} $tarball" | sha256sum --check --quiet || { + rm -f "$tarball" + error "$tool v$version download doesn't match its sha256" + } + mkdir -p "$dir" + tar -xzf "$tarball" -C "$dir" "$tool" + rm -f "$tarball" + log "Installed $tool v$version" +} + +main() { + parse_args "$@" + fetch_tool mdbook "$MDBOOK_VERSION" rust-lang/mdBook + fetch_tool mdbook-mermaid "$MERMAID_VERSION" badboy/mdbook-mermaid + export PATH="$TOOLS_DIR/mdbook-$MDBOOK_VERSION:$TOOLS_DIR/mdbook-mermaid-$MERMAID_VERSION:$PATH" + + cd "$ROOT_DIR" + # Writes mermaid.min.js and mermaid-init.js from the pinned release. + RUST_LOG=warn mdbook-mermaid install . >/dev/null + if [ "$SERVE" = true ]; then + mdbook serve + else + mdbook build + fi +} + +main "$@" diff --git a/scripts/lib.sh b/scripts/lib.sh index b31fd00..1690217 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -257,6 +257,12 @@ render_nginx_site() { "$INSTALL_DIR/deploy/nginx/relaysms-publisher-nginx.conf.template" >"$conf" } +# Renders the manual's nginx site for $1 into $2, serving $INSTALL_DIR/book. +render_docs_site() { + sed -e "s/__SERVER_NAME__/$1/g" -e "s|__BOOK_DIR__|$INSTALL_DIR/book|g" \ + "$INSTALL_DIR/deploy/nginx/relaysms-publisher-docs-nginx.conf.template" >"$2" +} + install_nginx_certbot() { if ! command -v nginx &>/dev/null || ! command -v certbot &>/dev/null; then log "Installing nginx and certbot" diff --git a/scripts/setup-observability.sh b/scripts/setup-observability.sh index 719e368..42a7d70 100755 --- a/scripts/setup-observability.sh +++ b/scripts/setup-observability.sh @@ -299,7 +299,7 @@ main() { fi case "$create_admin" in n | N | no | NO) - log "Skipping SigNoz admin account. OTLP ingestion stays blocked until one is created, see observability/README.md" + log "Skipping SigNoz admin account. OTLP ingestion stays blocked until one is created, see docs/operating/observability.md" ;; *) [ "$SIGNOZ_EMAIL_SET" = "1" ] || prompt SIGNOZ_EMAIL "SigNoz admin email [$SIGNOZ_EMAIL]: " "$SIGNOZ_EMAIL" @@ -351,7 +351,7 @@ main() { log "SigNoz: http://localhost:$SIGNOZ_PORT Uptime Kuma: http://localhost:$KUMA_PORT" [ -n "$SITE_NAME" ] && [ "$SKIP_NGINX" != "1" ] && log "SigNoz reverse proxy: $signoz_external_url" [ -n "$KUMA_SITE_NAME" ] && [ "$SKIP_NGINX" != "1" ] && log "Uptime Kuma reverse proxy: https://$KUMA_SITE_NAME" - log "Create Uptime Kuma monitors and set retention in SigNoz, see observability/README.md" + log "Create Uptime Kuma monitors and set retention in SigNoz, see docs/operating/observability.md" } main "$@" diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index a1ba40d..9320c64 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "relaysms-adapter-sdk" -version = "1.1.2" +version = "1.1.3" description = "Build RelaySMS Publisher platform adapters" readme = "README.md" license = "GPL-3.0-only" diff --git a/sdk/src/relaysms_adapter_sdk/adapter.py b/sdk/src/relaysms_adapter_sdk/adapter.py index d4e30ac..3095be3 100644 --- a/sdk/src/relaysms_adapter_sdk/adapter.py +++ b/sdk/src/relaysms_adapter_sdk/adapter.py @@ -49,7 +49,7 @@ def revoke(self, request: RevokeRequest) -> MaybeAwaitable[None]: def setup(cls, request: SetupRequest) -> MaybeAwaitable[SetupResult]: """Turn the [setup] inputs into settings, e.g. by registering an app. - A classmethod: it runs before the settings an instance reads exist. + A classmethod, since an adapter's __init__ may read the settings setup creates. """ raise AdapterError( f"{cls.__name__} has no setup; set its config instead.", diff --git a/sdk/src/relaysms_adapter_sdk/errors.py b/sdk/src/relaysms_adapter_sdk/errors.py index d791ab6..dc5af2b 100644 --- a/sdk/src/relaysms_adapter_sdk/errors.py +++ b/sdk/src/relaysms_adapter_sdk/errors.py @@ -12,7 +12,10 @@ class AdapterError(Exception): - """A failure the Publisher should report as is.""" + """A failure whose message reaches the Publisher. + + Any other exception reaches it only as INTERNAL_ERROR, with details in the log. + """ code: int = -32000 diff --git a/sdk/src/relaysms_adapter_sdk/runner.py b/sdk/src/relaysms_adapter_sdk/runner.py index bf5a013..ade278d 100644 --- a/sdk/src/relaysms_adapter_sdk/runner.py +++ b/sdk/src/relaysms_adapter_sdk/runner.py @@ -36,8 +36,9 @@ def handle( ) -> str: """Answer one JSON-RPC request line. - Given the class, it creates the adapter for the request, except for setup, - which runs on the class before the settings an adapter reads exist. + Given the class, it creates an adapter for each request, except setup, which + runs on the class because an adapter's __init__ may read the settings setup + creates. """ try: request = wire.parse_request(line) diff --git a/template.env b/template.env index b177f4b..6071082 100644 --- a/template.env +++ b/template.env @@ -1,7 +1,8 @@ # REST API HOST=127.0.0.1 PORT=16000 -# Worker processes for scripts/run.sh (default 1) and Docker (default 4). +# REST API worker processes. Default 1 under scripts/run.sh, 4 on Docker and +# server installs. # WORKERS= # gRPC @@ -20,7 +21,7 @@ NONCE_TTL_SECONDS=600 # Database DATABASE_DIALECT=sqlite -# Database Encryption (SQLCipher for SQLite, TDE for MySQL, disk-level for Postgres) +# Encrypt the SQLite file with SQLCipher. No effect on MySQL or PostgreSQL. DATABASE_ENCRYPTION_ENABLED=true # 64-char hex (32 bytes). Generate: openssl rand -hex 32 DATABASE_ENCRYPTION_KEY= @@ -30,7 +31,7 @@ DATABASE_FIELD_ENCRYPTION_ENABLED=true # 64-char hex (32 bytes). Generate: openssl rand -hex 32 DATABASE_FIELD_ENCRYPTION_KEY= -# Data Encryption Key (DEK) - used to encrypt all private keys +# Data encryption key: encrypts the server's private keys and keys the sender hash. # 64-char hex (32 bytes). Generate: openssl rand -hex 32 DATA_ENCRYPTION_KEY= @@ -55,7 +56,7 @@ SQLITE_DATABASE_PATH=data/relaysms.db # Broker/backend type: sqlite | redis | rabbitmq CELERY_BROKER_TYPE=sqlite # Worker concurrency. Leave unset to use the per-broker default -# (1 for sqlite, 4 for redis/rabbitmq). +# (1 for sqlite, else 4). # CELERY_WORKER_CONCURRENCY= # sqlite CELERY_BROKER_DB_PATH=data/celery_broker.db @@ -95,13 +96,13 @@ OFFLINE_PUBLISH_ALLOWED_PROTOCOLS=smtp OFFLINE_PUBLISH_SHARED_SECRET= # SMTP Ingestion Transport -# Mail body must be JSON: {"address": "", "text": ""} +# Mail body must be JSON: {"address": "", "text": ""} SMTP_TRANSPORT_ENABLED=false SMTP_IMAP_SERVER= SMTP_IMAP_PORT=993 SMTP_IMAP_USERNAME= SMTP_IMAP_PASSWORD= -# Comma-separated, e.g. INBOX,Spam - catches legit mail misfiled as spam +# Comma-separated, such as INBOX,Spam, to catch payloads misfiled as spam SMTP_IMAP_MAIL_FOLDER=INBOX # Optional mTLS client cert, only if the mail provider requires one SMTP_TLS_CLIENT_CERTIFICATE= @@ -112,7 +113,6 @@ SMTP_TLS_CLIENT_KEY= SMTP_ALLOWED_SENDERS= # authserv-id trusted for Authentication-Results verdicts (e.g. mx.google.com). # Required when SMTP_TRANSPORT_ENABLED=true -# unset rejects all mail. SMTP_TRUSTED_AUTHSERV_ID= SMTP_REQUIRE_DKIM=true SMTP_REQUIRE_SPF=true @@ -131,7 +131,7 @@ PLATFORMS_ADAPTERS_STATE_DIR=data/platforms/state PLATFORMS_GITHUB_ORGS= # Auth (credentials are managed with ./publisher.sh creds) -# Comma-separated web client origins, e.g. https://console.example.org. Must be +# Comma-separated web client origins, e.g. https://console.example.com. Must be # the same site as the API. Empty if same origin. AUTH_WEB_ORIGINS= # false only for local http development. @@ -146,7 +146,7 @@ API_DOCS_ENABLED=false LOG_LEVEL=info # OpenTelemetry. Optional, blank disables instrumentation. See -# observability/README.md. +# docs/operating/observability.md. OTEL_EXPORTER_OTLP_ENDPOINT= OTEL_EXPORTER_OTLP_PROTOCOL=grpc OTEL_TRACES_SAMPLER=parentbased_traceidratio @@ -155,6 +155,6 @@ OTEL_LOGS_EXPORTER=otlp OTEL_PYTHON_LOG_CORRELATION=true # Uptime Kuma push-monitor heartbeats. Blank disables. See -# observability/README.md. +# docs/operating/observability.md. UPTIME_KUMA_WORKER_PUSH_URL= UPTIME_KUMA_SMTP_PUSH_URL= diff --git a/tests/cli/test_reference.py b/tests/cli/test_reference.py new file mode 100644 index 0000000..9b9ce27 --- /dev/null +++ b/tests/cli/test_reference.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: GPL-3.0-only + +import re + +import click + +from publisher import config +from publisher.cli import cli + + +def doc_section(page: str, heading: str) -> str: + text = (config.ROOT / "docs" / page).read_text() + return text.split(f"## {heading}", 1)[1].split("\n## ", 1)[0] + + +def commands(group: click.Group, prefix: str = "") -> set[str]: + names = set() + for name, command in group.commands.items(): + if isinstance(command, click.Group): + names |= commands(command, f"{prefix}{name} ") + else: + names.add(f"{prefix}{name}") + return names + + +def test_cli_reference_lists_every_publisher_command(): + section = doc_section("reference/cli.md", "publisher.sh") + documented = set() + for group, cells in re.findall(r"^\| `([a-z-]+)` \| ([^|]+) \|", section, re.M): + documented |= {f"{group} {name}" for name in re.findall(r"`([a-z -]+)`", cells)} + + assert commands(cli) - documented == set(), "missing from docs/reference/cli.md" + assert documented - commands(cli) == set(), "in docs/reference/cli.md, not the CLI" + + +def test_cli_reference_lists_every_manage_command(): + usage = re.search( + r"Usage: \$0 \{([^}]+)\}", (config.ROOT / "manage.sh").read_text() + ) + assert usage + actual = set(usage[1].split("|")) + section = doc_section("operating/services.md", "manage.sh") + documented = set() + for cell in re.findall(r"^\| (`[^|]+) \|", section, re.M): + documented |= set(re.findall(r"`([a-z]+)[` ]", cell)) + + assert actual - documented == set(), "missing from docs/operating/services.md" + assert documented - actual == set(), "in docs/operating/services.md, not manage.sh" diff --git a/tests/scripts/test_build_book.py b/tests/scripts/test_build_book.py new file mode 100644 index 0000000..afdddaa --- /dev/null +++ b/tests/scripts/test_build_book.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: GPL-3.0-only + +import os +import stat +import subprocess + +import pytest + + +@pytest.fixture +def build(install, tmp_path): + """Run build-book.sh with stub curl and uname, so nothing is downloaded.""" + stubs = tmp_path / "bin" + stubs.mkdir() + for name, body in { + # Called as: curl -fsSL -o FILE URL + "curl": 'touch "$CURL_CALLED"; echo junk >"$3"', + "uname": 'echo "$ARCH"', + }.items(): + stub = stubs / name + stub.write_text(f"#!/usr/bin/env bash\n{body}\n") + stub.chmod(stub.stat().st_mode | stat.S_IEXEC) + + def run(arch: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", "scripts/build-book.sh"], + cwd=install, + env={ + **os.environ, + "NO_COLOR": "1", + "PATH": f"{stubs}:{os.environ['PATH']}", + "ARCH": arch, + "CURL_CALLED": str(tmp_path / "curl-called"), + }, + capture_output=True, + text=True, + check=False, + ) + + return run + + +class TestBuildBook: + def test_refuses_a_download_that_fails_its_checksum(self, install, build): + result = build("x86_64") + assert result.returncode != 0 + assert "doesn't match its sha256" in result.stderr + assert not (install / ".mdbook" / "mdbook-0.5.3" / "mdbook").exists() + + def test_refuses_an_architecture_without_a_release(self, tmp_path, build): + result = build("riscv64") + assert result.returncode != 0 + assert "No mdbook release for riscv64" in result.stderr + assert not (tmp_path / "curl-called").exists() diff --git a/tests/test_config.py b/tests/test_config.py index c2337ed..524ecb8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -31,6 +31,11 @@ def errors(**env): return "\n".join(config.check({**BASE_ENV, **env})) +def template_env_names(): + text = (config.ROOT / "template.env").read_text() + return set(re.findall(r"^\s*(?:#\s*)?([A-Za-z_]\w*)=", text, re.M)) + + def test_defaults(): assert load(LoggingConfig).log_level == "INFO" assert not load(GrpcConfig).tls_enabled @@ -224,8 +229,7 @@ def test_template_env_paths_match_defaults(): def test_template_env_lists_every_variable_config_reads(): # sync-env.sh --prune drops what the template lacks, so a variable config # reads must be in it, if only commented out. - text = (config.ROOT / "template.env").read_text() - template = set(re.findall(r"^\s*(?:#\s*)?([A-Za-z_]\w*)=", text, re.M)) + template = template_env_names() # An empty environment that records every name config asks for. env = Mock() env.get.side_effect = lambda key, default=None: default @@ -242,6 +246,20 @@ def test_template_env_lists_every_variable_config_reads(): assert unused == set(), "in template.env but unused" +def test_settings_reference_matches_template_env(): + template = template_env_names() + reference = (config.ROOT / "docs" / "configuration" / "settings.md").read_text() + # Only the first column, so a name mentioned in a description doesn't count. + documented = { + name + for row in re.findall(r"^\|\s*(`[A-Z].*?)\s*\|", reference, re.M) + for name in re.findall(r"`([A-Z][A-Z0-9_]*)`", row) + } + + assert template - documented == set(), "missing from the settings reference" + assert documented - template == set(), "in the settings reference, not template.env" + + def test_auth_session_defaults(): auth = load(AuthConfig)