diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 3b7740a..54ec57b 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -20,8 +20,6 @@ runs: sudo apt-get update sudo apt-get install -y --no-install-recommends libsqlcipher-dev libmagic1 - # A hit skips pip entirely. The venv links to its interpreter, so the key - # has the exact Python version. - uses: actions/cache@v6 id: venv-cache with: @@ -40,7 +38,6 @@ runs: shell: bash run: echo "sha=$(git rev-parse HEAD:lib_relaysms_payload_specs)" >> "$GITHUB_OUTPUT" - # The bindings don't depend on the Python version, so every job shares them. - uses: actions/cache@v6 id: specs-cache with: diff --git a/.github/workflows/check-requirements.yml b/.github/workflows/check-requirements.yml index 4b8e9fb..3bd3de3 100644 --- a/.github/workflows/check-requirements.yml +++ b/.github/workflows/check-requirements.yml @@ -28,7 +28,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip - # Dry run so conflicts fail the PR without needing the native build deps. + # Resolves the requirements without building them. - name: Resolve dependencies run: | pip install --dry-run --disable-pip-version-check \ diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 271bae9..ca8777b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -41,7 +41,7 @@ jobs: tests: name: Tests (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest - # Ubuntu 24.04 LTS and Debian 13. The coverage job covers 3.14, the Docker image's. + # The Python versions of Ubuntu 24.04 and Debian 13. The coverage job runs 3.14, the Docker image's. strategy: fail-fast: false matrix: @@ -105,8 +105,6 @@ jobs: with: fetch-depth: 0 - # Adapters pin SDK releases while the Publisher runs sdk/ as it is, so every - # change to it needs a version that can be tagged. - name: Require a version bump for SDK changes env: BASE: ${{ github.event.pull_request.base.sha || github.event.before }} @@ -114,7 +112,7 @@ jobs: git cat-file -e "$BASE^{commit}" 2>/dev/null || exit 0 git diff --quiet "$BASE...HEAD" -- sdk/src && exit 0 git diff "$BASE...HEAD" -- sdk/pyproject.toml | grep -q '^+version = ' && exit 0 - echo "::error file=sdk/pyproject.toml::sdk/src changed; bump version here." + echo "::error file=sdk/pyproject.toml::sdk/src changed without a new version here." exit 1 deploy-staging: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c3372fa..600b2f7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v7 - # Local files and anchors only, so an external site being down can't fail a PR. + # Checks local files and anchors, not external sites. - name: Check Markdown links uses: lycheeverse/lychee-action@v2 with: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f666dfc..7091553 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -47,7 +47,6 @@ jobs: with: python-version: "3.12" - # The e2e tests only need pytest, not the app's test environment. - name: Install pytest run: | python -m venv venv diff --git a/.gitignore b/.gitignore index 2ab87d2..53f8cea 100644 --- a/.gitignore +++ b/.gitignore @@ -129,11 +129,9 @@ venv/ # Written by install.sh into the install directory .instance-name -# Runtime data: adapters, their venvs and assets, gateway client imports +# Runtime data: adapters' code, venvs, config and state /data/ -# Runtime data an install kept here before data/ existed. It can hold adapter -# credentials and gateway client numbers, so it stays ignored until -# scripts/migrate-runtime-data.sh moves it. +# Where older installs kept runtime data. It can hold adapter credentials and gateway client numbers. /platforms/ /gateway_clients/ ENV/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9183792..479a96e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,47 +1,71 @@ # Contributing +How to change the Publisher's code, tests and manual. To install or run it, read the [manual](docs/README.md) instead. + ## Contents -- [Ways to contribute](#ways-to-contribute) -- [Reporting security issues](#reporting-security-issues) +- [Where to start](#where-to-start) +- [Your first pull request](#your-first-pull-request) - [Development setup](#development-setup) -- [Workflow](#workflow) +- [Run the checks CI runs](#run-the-checks-ci-runs) +- [Testing](#testing) - [Project layout](#project-layout) - [Code conventions](#code-conventions) -- [Testing](#testing) - [Common changes](#common-changes) - [Documentation](#documentation) -- [When CI fails](#when-ci-fails) +- [Commits and pull requests](#commits-and-pull-requests) - [AI-assisted contributions](#ai-assisted-contributions) - [License](#license) -## Ways to contribute +## Where to start + +| You want to | Do this | +| --- | --- | +| Report a vulnerability | Don't open an issue. Follow [SECURITY.md](SECURITY.md). | +| Report a bug | Search the [issues](https://github.com/smswithoutborders/RelaySMS-Publisher/issues), then open one with the details below. | +| Suggest a feature | Open an issue that describes the problem before the solution. | +| Change the payload format, the gRPC or REST contract, or the database schema | Agree on the design in an issue before writing code. | +| Fix a typo or a clear bug | Open a pull request with a test. Comment on the issue first for anything larger. | +| Change a platform adapter | Open it in the adapter's own repository. See [Platform adapters](#platform-adapters). | +| Change the payload-specs library | Open it in [rusty_relaysms_payload_specs](https://github.com/smswithoutborders/rusty_relaysms_payload_specs). This repository only pins a commit of it. | -**Report a bug.** Search the [issues](https://github.com/smswithoutborders/RelaySMS-Publisher/issues) first. A useful report has: +A useful bug report has: -- what you did, what you expected and what happened instead; -- the smallest steps or payload that reproduce it; -- the commit or release, Python version, database and how you run the services (systemd, Docker, `make run`); -- relevant log lines, with tokens, keys and phone numbers removed. +- what you did, what you expected and what happened instead +- the smallest steps or payload that reproduce it +- the commit or release, Python version, database and how you run the services (systemd, Docker or `make run`) +- relevant log lines, with tokens, keys and phone numbers removed -**Suggest a feature.** Open an issue that describes the problem before the solution. For anything that changes the payload format, the gRPC or REST contract, or the database schema, agree on the design in the issue before writing code. +## Your first pull request -**Fix something.** Small fixes (typos, clear bugs with a test) can go straight to a pull request. For larger changes, comment on the issue first. +1. Install the [prerequisites](#prerequisites), then fork, clone and set up as in [Install](#install). +2. Branch from the latest upstream `staging`: -## Reporting security issues + ```bash + git fetch upstream + git switch -c fix/ upstream/staging + ``` -> [!CAUTION] -> Don't open a public issue for a vulnerability. See [SECURITY.md](SECURITY.md). +3. Make the change, with a test that fails without it. +4. Run what CI runs: + + ```bash + git add # hooks only see tracked files + make check test + ``` + +5. Commit with a [Conventional Commits](#commits) message, push to your fork and open a pull request against `staging`. ## Development setup ### Prerequisites - Python 3.12 or newer -- System packages (Ubuntu): +- System packages (Ubuntu or Debian): ```bash - sudo apt install python3-dev build-essential libsqlcipher-dev libmagic1 pkg-config make git + sudo apt install python3-dev python3-venv build-essential pkg-config \ + libsqlcipher-dev libmagic1 openssl make git curl ``` - Rust and cargo, to build the payload-specs library. Install them with [rustup](https://rustup.rs) if `cargo --version` fails: @@ -51,44 +75,48 @@ source "$HOME/.cargo/env" ``` +- [Podman](https://podman.io), rootless, only for `make test-dialects` and `make test-e2e` + ### Install -[Fork the repository](https://github.com/smswithoutborders/RelaySMS-Publisher/fork) on GitHub, then clone your fork and add this repository as `upstream`: +[Fork the repository](https://github.com/smswithoutborders/RelaySMS-Publisher/fork), then: ```bash -git clone --recurse-submodules git@github.com:/RelaySMS-Publisher.git +git clone --recurse-submodules git@github.com:/RelaySMS-Publisher.git cd RelaySMS-Publisher git remote add upstream git@github.com:smswithoutborders/RelaySMS-Publisher.git make setup ``` -`make setup` creates `venv` with the dev dependencies and pre-commit hooks, runs `make build`, creates `.env` with fresh encryption keys, checks the config and migrates the database. Running it again keeps an existing `.env` as is; `./sync-env.sh` adds the variables it's missing. +`make setup` creates `venv` with the dev dependencies, installs the Git hooks, runs `make build`, creates `.env` with fresh encryption keys, checks the config and migrates the database. Running it again keeps an existing `.env`. After a pull adds settings to `template.env`, `./sync-env.sh` copies them into `.env` without changing existing values. -Make targets run `venv/bin/python`. Run `source venv/bin/activate` before the `python` commands below. +Make targets use `venv/bin/python`. Run `source venv/bin/activate` before the `python` commands below. > [!NOTE] -> `make build` generates files that are gitignored. Run it again after pulling changes to `protos/` or the `lib_relaysms_payload_specs` submodule. +> `make build` generates gitignored files. Run it again after pulling changes to `protos/` or the `lib_relaysms_payload_specs` submodule. ### Run locally ```bash make creds # administrator "dev", prints the password once -make run # gRPC, REST, SMTP, worker and beat in the foreground +make run # every service in the foreground ``` -Log in to the REST API with that username and password, e.g. `curl -u dev: http://127.0.0.1:16000/v1/auth/me` with the default `PORT`. +`make run` starts gRPC, REST, the Celery worker and beat, and the SMTP listener when `SMTP_TRANSPORT_ENABLED=true`. Check your login against the REST API on the default `PORT`: -Each service can also run on its own: +```bash +curl -u dev: http://127.0.0.1:16000/v1/auth/me +``` -| Service | Command | +| To | Run | | --- | --- | -| REST API | `uvicorn publisher.api.rest.app:app` | -| gRPC | `python -m publisher.api.grpc` | -| SMTP | `python -m publisher.smtp` | -| Celery worker and beat | `celery -A publisher.tasks.celery_app:celery_app worker` / `beat` | -| CLI | `python -m publisher --help` (or `./publisher.sh`) | - -`python -m tools.client` drives the gRPC and REST flows by hand, see [docs/developers/reference-client.md](docs/developers/reference-client.md). +| Run the REST API alone | `uvicorn publisher.api.rest.app:app` | +| Run gRPC alone | `python -m publisher.api.grpc` | +| Run SMTP alone | `python -m publisher.smtp` | +| Run the Celery worker or beat alone | `celery -A publisher.tasks.celery_app:celery_app worker` (or `beat`) | +| Use the CLI | `python -m publisher --help` | +| Fill the database with sample data | `python -m publisher seed --help` | +| Drive the gRPC and REST flows by hand | `python -m tools.client`, see [the reference client](docs/developers/reference-client.md) | ### Make targets @@ -101,87 +129,101 @@ Each service can also run on its own: | `make migrate` | Apply Alembic migrations | | `make creds` | Create an administrator credential, `dev` unless `CREDS_USER` is set | | `make run` | Start every service | +| `make check` | Run every pre-commit hook on all files | | `make test` | Run pytest | -| `make coverage` | Run pytest and list the lines no test runs | +| `make coverage` | Run pytest, list untested lines, and fail under the `fail_under` in `pyproject.toml` | +| `make test-sdk` | Test the adapter SDK in `sdk/` | +| `make test-dialects` | Migrate and query every supported database (podman) | +| `make test-e2e` | Install, update and uninstall in systemd containers (podman) | | `make docs` | Regenerate `docs/openapi.json` from the REST app | | `make book` | Build the manual in `docs/` into `book/` | | `make book-serve` | Preview the manual on , rebuilding on changes | -| `make test-dialects` | Migrate and query SQLite, SQLCipher, Postgres, MySQL and MariaDB (podman) | -| `make test-e2e` | Install, update and uninstall in a systemd container (podman) | -| `make check` | Run every pre-commit hook on all files | | `make clean` | Remove generated code | -## Workflow +Every pytest target passes `PYTEST_ARGS` on, such as `make test PYTEST_ARGS="-k token -x"`. -### Branches +## Run the checks CI runs -- Branch from `staging` and open pull requests against `staging`. -- `staging` deploys to the staging server on every push. Put `[migrate]` in a commit message to also run the migrations, e.g. `feat(reports): add a reports table [migrate]`. -- `staging` is merged into `main` for a release. - -Name branches after the change, such as `fix/help-text-typo` or `feat/export-csv`. Start each one from the latest upstream `staging`, push it to your fork and open the pull request from there: - -```bash -git fetch upstream -git switch -c fix/help-text-typo upstream/staging -# ...commit... -git push -u origin fix/help-text-typo -``` +`make setup` installs three Git hooks: the pre-commit checks on `git commit`, commitizen on the commit message, and the test suite on `git push`. -If `staging` moves on while your PR is open, rebase onto it and force-push: +| CI check | Runs on | Run it locally | +| --- | --- | --- | +| **Hooks** (formatting, lint and types, see [.pre-commit-config.yaml](.pre-commit-config.yaml)) | Every PR | `make check` | +| **Tests** on Python 3.12 and 3.13 | Every PR | `make test` | +| **Coverage** on Python 3.14 | Every PR | `make coverage` | +| **SDK** on Python 3.12 to 3.14 | Every PR | `make test-sdk` | +| **SDK version** | Every PR | Bump `version` in `sdk/pyproject.toml` when `sdk/src` changes | +| **Database dialects** | PRs touching the database code, migrations or requirements | `make test-dialects` | +| **E2E** (Ubuntu and Debian) | PRs touching the scripts, `deploy/`, migrations or requirements | `make test-e2e` | +| **Docs** (link check and book build) | PRs touching Markdown or the book | `make book` | +| **Check Requirements** | PRs touching `requirements*.txt` | `pip install --dry-run -r requirements-observability.txt` | -```bash -git fetch upstream -git rebase upstream/staging -git push --force-with-lease -``` +The workflows in [.github/workflows](.github/workflows) list the exact paths. **Database dialects** and **E2E** can also run from the Actions tab with **Run workflow**. -> [!TIP] -> Leave "Allow edits by maintainers" ticked. +Each pytest job except **SDK** adds a results table to the run's summary page, with the failing tests under it. -### Commits +| When this fails | Try | +| --- | --- | +| Ruff or a whitespace hook | It fixed the files in place. Re-add them and commit. | +| Pyright | Fix the type, or narrow it with an `assert`. Use `# pyright: ignore[]` only with a reason. | +| import-linter | A lower layer imports a higher one, see [Project layout](#project-layout). Move the code down a layer or pass the dependency in as an argument. | +| Tests pass on newer Pythons but fail on 3.12 | Look for newer syntax or an unquoted forward reference. See [Checking Python 3.12](#checking-python-312). | +| Coverage | Test the lines the report lists. | +| A dialect | `make test-dialects PYTEST_ARGS="-k mysql"` | +| A distro in E2E | `make test-e2e PYTEST_ARGS="-k debian"` | +| Check Requirements | The pinned dependencies conflict. The dry run above shows which. | -Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/), checked by commitizen in the `commit-msg` hook: +## Testing +```bash +make test # whole suite +python -m pytest tests/api/rest -q # one directory +python -m pytest -k token_id -x # by name, stop at the first failure ``` -(): - +Tests run against in-memory SQLite. `tests/conftest.py` sets the environment before `publisher.config` is imported, so your local `.env` is ignored. - -``` +| Directory | Covers | Run with | +| --- | --- | --- | +| `tests/` (mirroring `publisher/`) | The Python package | `make test` | +| `tests/scripts` | The shell scripts | `make test` | +| `tests/dialects` | Migrations and queries on SQLite, SQLCipher, Postgres, MySQL and MariaDB | `make test-dialects` | +| `tests/e2e` | Install, update and uninstall on Ubuntu 24.04 and Debian 13 | `make test-e2e` | -Keep it to the summary line unless the change needs explaining: +`make test-dialects` starts the database servers in rootless podman and removes them afterwards. It skips MariaDB on a host with an AppArmor profile for `mariadbd`, which stops the server in the container. Add a dialect test when a query or migration could behave differently on another database. -``` -fix(cli): show the missing file when a command fails -feat(reports)!: rename REPORTS_DIR to REPORTS_PATH -``` +`make test-e2e` installs the working tree, uncommitted changes included, into systemd containers. It needs network access and takes a few minutes. -Common types are `feat`, `fix`, `refactor`, `test`, `docs`, `build`, `ci` and `chore`. Scopes name the area, such as `grpc`, `rest`, `auth`, `keys` or `manage`. Mark breaking changes with `!` (`feat(reports)!: ...`) and explain the migration path in a `BREAKING CHANGE:` footer. Breaking changes include removed or renamed env vars, CLI commands, endpoints, RPC fields and anything an operator must do by hand on update. +### Writing tests -### Hooks +- Put a test where its module lives: `publisher/api/rest/v1/stats.py` is tested in `tests/api/rest/test_stats.py`. +- Name tests after the behavior, such as `test_list_skips_disabled_items`, not the function. +- Test through the public interface (a route, an RPC, a domain function). Patch only what crosses the process boundary, such as adapter IPC or outbound HTTP. +- A module that touches the database opts in at the top: -`pre-commit install` sets up three stages: + ```python + pytestmark = pytest.mark.usefixtures("test_db", "fast_hasher") + ``` -- **pre-commit:** whitespace and line-ending fixes, YAML/TOML checks, ruff lint and format, shellcheck and shfmt for shell scripts, hadolint and dockerfmt for Dockerfiles, dclint for Compose files, pyright and import-linter; -- **commit-msg:** commitizen; -- **pre-push:** the test suite. +Shared fixtures live in `tests/fixtures.py` and `tests/conftest.py`, and helpers in `tests/helpers.py`. The ones most tests use: -> [!TIP] -> Hooks only see tracked files, so `git add` new files before running `make check`. +| Name | Gives | +| --- | --- | +| `test_db` | A fresh in-memory database with every table | +| `fast_hasher` | A cheap password hasher, for credential tests | +| `client`, `admin` | A `TestClient` for the REST app, and Basic auth headers for the administrator | +| `set_config` | Changes a config section bound in a module, for one test | +| `adapter_calls` | Fakes adapter processes: set `results[method]` to a result or an error, read `calls` back | -### Pull request checklist +### Checking Python 3.12 -- [ ] The PR targets `staging` and does one thing. -- [ ] `make check` and `make test` pass locally. -- [ ] New behavior and bug fixes come with tests. A bug fix has a test that fails without it. -- [ ] The manual in `docs/` and `template.env` are updated if behavior, settings or commands changed. -- [ ] Schema changes include a migration, and a commit message with `[migrate]`. -- [ ] The description says what changed, why, and how it was tested. Link the issue with `Closes #123`. -- [ ] Anything operators must do on deploy is called out. +With [uv](https://docs.astral.sh/uv/): -A maintainer reviews every PR. Address comments with new commits; they're squashed or tidied on merge as needed. +```bash +uv venv --python 3.12 .venv312 +uv pip install --python .venv312 -r requirements-dev.txt +make build test PYTHON=.venv312/bin/python +``` ## Project layout @@ -191,135 +233,65 @@ A maintainer reviews every PR. Address comments with new commits; they're squash | `publisher/tasks` | Celery tasks, queued by the entry points and run by the worker | | `publisher/keys.py`, `publications.py`, `tokens.py`, `credentials.py`, `platforms/`, `gateway_clients/` | Domain logic | | `publisher/config.py`, `db/`, `models/`, `log.py`, `errors.py`, `crypto.py` | Infrastructure | -| `deploy/` | systemd units and nginx templates | -| `scripts/` | Install, setup and maintenance scripts used by `manage.sh` | +| `sdk/` | The adapter SDK, released on its own | | `migrations/` | Alembic migrations | | `protos/` | gRPC service definitions | +| `deploy/` | systemd units and nginx templates | +| `scripts/` | Shell scripts for installing, running and maintaining the Publisher, and the shared `lib.sh` | | `tools/` | The reference client | -| `data/` | Runtime data: adapter files (not tracked) | -| `tests/` | Tests, see [Testing](#testing) | - -Each code row may import only the rows below it, never the other way round, and entry points don't import each other. import-linter enforces this in `make check`. - -### How a publication flows +| `data/` | Runtime data, not tracked | -1. A payload arrives over REST (`/v1/publications`) or SMTP. -2. The entry point validates it and calls `queue_publication`, which replaces the sender's address with a keyed hash and queues `publish_message`. -3. The worker runs `publisher/tasks/publication_task.py`, which in one transaction calls `publications.prepare` (deletes the slot's keys and decrypts), `send` (runs the adapter through `platforms/ipc.py`) and `finish` (stores a refreshed token). -4. It commits only after a successful send, so a failure leaves the keys usable, and a duplicate payload waits on their row lock. -5. The task records the outcome in the publication stats. - -The gRPC service handles the account side: storing OAuth2 and PNBA tokens, revoking them and syncing key pools. +Each `publisher` row may import only the rows below it, and `cli`, `api` and `smtp` don't import each other. import-linter enforces this in `make check`. The [architecture](docs/explanation/architecture.md) page shows how the services fit together and how a publication flows. ## Code conventions -The hooks enforce formatting, lint, types and commit messages. These rules are checked in review: +The hooks enforce formatting, lint, types and commit messages. Review checks these: -- **Functions by default.** Modules group related behavior. Use a class only for real state (a cache, a connection) or when a framework needs one (ORM models, structs, interceptors, servicers, config sections). +- **Functions by default.** Modules group related behavior. Use a class only for real state (a cache, a connection) or when a framework needs one (ORM models, interceptors, servicers, config sections). - **Dependencies are arguments.** Domain and model functions take the database `session` first, then anything else they need, such as the acting credential. -- **The entry point owns the transaction.** A route, gRPC handler, task or CLI command opens the session; code below it never opens a session or commits. -- **No transaction stays open while an adapter runs,** since a call can take a minute. Publishing is the one exception (see above). +- **The entry point owns the transaction.** A route, gRPC handler, task or CLI command opens the session. Code below it never opens a session or commits. +- **No transaction stays open while an adapter runs,** since a call can take a minute. - **Domain errors subclass `PublisherError`** (`publisher/errors.py`). Each interface maps them to its own responses: HTTP status codes, gRPC status codes, SMTP replies. - **Loggers are `logging.getLogger(__name__)`.** Each entry point calls `publisher.log.setup_logging()` once. Never log tokens, keys, passwords or decrypted content. -- **Comments explain why, not what.** Docstrings are one line unless callers need a `Raises:` section. REST route docstrings become the API reference, so describe what a client needs there. +- **Comment only what the code can't say.** REST route docstrings become the API reference, so describe what a client needs there. - **Mark overridden framework methods with `@typing.override`.** - **Code must run on Python 3.12.** Quote `TYPE_CHECKING` forward references (`Mapped["Token"]`) and avoid syntax newer than 3.12. - **Every `.py` file starts with** `# SPDX-License-Identifier: GPL-3.0-only`. -Shell scripts follow one layout: `#!/usr/bin/env bash`, the SPDX line, `set -Eeuo pipefail`, then `source` `scripts/lib.sh` for logging, errors and shared helpers. Scripts with options parse them in `parse_args` and run from `main "$@"`. `install.sh` is the one exception: piped through curl, it clones the repo first and re-runs from the clone. - -## Testing - -```bash -make test # whole suite -python -m pytest tests/api/rest -q # one directory -python -m pytest -k token_id -x # by name, stop at first failure -make coverage # whole suite, with untested lines -``` - -Tests run against in-memory SQLite. `tests/conftest.py` sets the environment before `publisher.config` is imported, so your local `.env` is ignored. - -| Directory | Covers | Runs in | -| --- | --- | --- | -| `tests/` (mirroring `publisher/`) | The Python package | `make test` | -| `tests/scripts` | The shell scripts, run with bash | `make test` | -| `tests/dialects` | Migrations and queries on SQLite, SQLCipher, Postgres 16, MySQL 8.0 and MariaDB 10.11 | `make test-dialects` | -| `tests/e2e` | Install, update and uninstall on Ubuntu 24.04 and Debian 13 | `make test-e2e` | - -`make test-dialects` starts Postgres, MySQL and MariaDB in rootless podman and removes them afterwards; `PYTEST_ARGS="-k mysql"` runs one dialect. MariaDB is skipped on a machine with MariaDB installed, since its AppArmor profile stops the container's server. Add a test there when a query or migration could behave differently on another database. - -`make test-e2e` installs the working tree, uncommitted changes included, into systemd containers. It needs rootless podman and network access, and takes a few minutes; `PYTEST_ARGS="-k debian"` runs one distro. - -### Writing tests - -- Put a test where its module lives: `publisher/api/rest/v1/stats.py` is tested in `tests/api/rest/test_stats.py`. -- Name tests after the behavior, such as `test_list_skips_disabled_items`, not the function. -- Test through the public interface (a route, an RPC, a domain function). Patch only what crosses the process boundary, such as adapter IPC or outbound HTTP. - -### Fixtures and helpers - -| Name | Where | Gives | -| --- | --- | --- | -| `test_db` | `tests/fixtures.py` | A fresh in-memory database with every table | -| `fast_hasher` | `tests/fixtures.py` | A cheap password hasher, for credential tests | -| `app`, `client` | `tests/fixtures.py` | The FastAPI app and a `TestClient` for it | -| `password`, `admin` | `tests/fixtures.py` | The administrator `USERNAME`'s password, and Basic auth headers for it | -| `platforms_config` | `tests/fixtures.py` | Adapter directories under `tmp_path`; call it to change other `PlatformsConfig` fields | -| `set_config` | `tests/conftest.py` | Changes a config section bound in a module, for one test | -| `USERNAME`, `create_credential`, `basic_auth`, `login`, `can_log_in` | `tests/helpers.py` | Credential and auth helpers | -| `get_etag` | `tests/helpers.py` | GETs a URL and returns its `ETag`, for `If-Match` | -| `add_adapter`, `link_account` | `tests/helpers.py` | An adapter row, and a token linked through it | -| `adapter_calls` | `tests/fixtures.py` | Fakes adapter processes: set `results[method]` to a result or an error, read `calls` back | -| `fake_adapter_build` | `tests/fixtures.py` | Installs adapters without running pip | - -Modules that touch the database opt in at the top: - -```python -pytestmark = pytest.mark.usefixtures("test_db", "fast_hasher") -``` - -### Checking Python 3.12 - -CI runs the suite on 3.12, 3.13 and 3.14. To check 3.12 locally with [uv](https://docs.astral.sh/uv/): - -```bash -uv venv --python 3.12 .venv312 -uv pip install --python .venv312 -r requirements-dev.txt -make build test PYTHON=.venv312/bin/python -``` +Shell scripts follow one layout: `#!/usr/bin/env bash`, the SPDX line, `set -Eeuo pipefail`, then `source` `scripts/lib.sh` for logging, errors and shared helpers. Scripts with options parse them in `parse_args` and run from `main "$@"`. ## Common changes ### Add a setting 1. Add the field to its section in `publisher/config.py` and read it in that section's `load`. -2. Add it, commented, to `template.env` and to [docs/configuration/settings.md](docs/configuration/settings.md); a test fails until both list it. +2. Add it, commented, to `template.env` and to [the settings reference](docs/configuration/settings.md). A test fails until both list it. 3. Check validation with `python -m publisher config check`, and add a test in `tests/test_config.py`. ### Change the database schema 1. Change the model in `publisher/models/`. Import a new model in `publisher/models/__init__.py`, or tests won't create its table. -2. Add `migrations/versions/NNN_short_slug.py` with the next number as `revision` and the previous one as `down_revision` (`"042"` after `"041"`). Write both `upgrade` and `downgrade`. -3. Run `make migrate`, then `python -m alembic downgrade -1` and `make migrate` again, on SQLite, and on MySQL or PostgreSQL if the migration uses dialect-specific features. +2. Add `migrations/versions/NNN_short_slug.py` with the next number as `revision` and the previous one as `down_revision`. Write both `upgrade` and `downgrade`. +3. Run `make migrate`, `python -m alembic downgrade -1` and `make migrate` again. Run `make test-dialects` if the migration uses dialect-specific features. 4. Put `[migrate]` in the commit message so the staging deploy applies it. ### Add a REST endpoint -1. Add the route to the matching module in `publisher/api/rest/v1/` (or a new module with its own `APIRouter`, registered in `routes.py`). +1. Add the route to the matching module in `publisher/api/rest/v1/`, or a new module with its own `APIRouter` registered in `routes.py`. 2. Put request and response models in `schemas.py`. Use `Depends(get_db)` for the session, and `Security(authorize, scopes=[...])` if it needs a login. A new scope goes in `Scope` in `publisher/models/credential.py`. -3. Map domain errors to `ApiError` (`errors.py`) with a message a client can show, and put ids and other detail in its `log=`. -4. Test it in `tests/api/rest/`, describe it in its docstring and run `make docs`. Update [docs/reference/rest-api.md](docs/reference/rest-api.md) only for what the reference doesn't show. +3. Map domain errors to `ApiError` (`v1/errors.py`) with a message a client can show, and put ids and other detail in its `log=`. +4. Test it in `tests/api/rest/`, describe it in its docstring and run `make docs`. Update [the REST reference](docs/reference/rest-api.md) only for what the generated reference doesn't show. ### Change the gRPC API -1. Edit `protos/v3/publisher.proto` and run `make protos`. -2. Add a module per RPC in `publisher/api/grpc/v3/` and a thin method on the servicer that calls it. -3. Raise `grpc_interceptor` exceptions such as `InvalidArgument` or `Unauthenticated`; `ErrorInterceptor` turns them into status codes and anything else into `INTERNAL`. -4. Test it in `tests/api/grpc/` and update [docs/reference/grpc-api.md](docs/reference/grpc-api.md). - > [!WARNING] > Deployed clients depend on the proto: only add fields. +1. Edit `protos/v3/publisher.proto` and run `make protos`. +2. Add a module per RPC in `publisher/api/grpc/v3/` and a thin method on the servicer that calls it. +3. Raise `grpc_interceptor` exceptions such as `InvalidArgument` or `Unauthenticated`. `ErrorInterceptor` turns them into status codes and anything else into `INTERNAL`. +4. Test it in `tests/api/grpc/` and update [the gRPC reference](docs/reference/grpc-api.md). + ### Add a CLI command Add a `click` command to the matching group in `publisher/cli/`, or a new module registered in `publisher/cli/__init__.py`. Open the session in the command and pass it down. @@ -330,7 +302,9 @@ Add it under `publisher/tasks/`. Keep the explicit `tasks.*` name so queued mess ### Platform adapters -Adapters live in their own repositories and are built with the SDK in `sdk/` ([sdk/README.md](sdk/README.md)). Their CI runs `.github/workflows/adapter.yml` at the SDK tag they pin, so a change to it reaches an adapter only when it moves its pin. Install one locally with `python -m publisher platforms add `; on a server use `./publisher.sh platforms`, which runs as the service user. See [docs/platforms/manage.md](docs/platforms/manage.md). +Adapters live in their own repositories and are built with the SDK in `sdk/`, see [sdk/README.md](sdk/README.md) and [Writing an adapter](docs/developers/writing-an-adapter.md). Install one locally with `python -m publisher platforms add `. + +A change to `sdk/src` needs a new `version` in `sdk/pyproject.toml`. Adapters pin SDK releases and run `.github/workflows/adapter.yml` at the tag they pin, so a change reaches an adapter only when it moves its pin. ## Documentation @@ -340,22 +314,61 @@ The manual in `docs/` is an [mdBook](https://rust-lang.github.io/mdBook/) that r - List a new page in `docs/SUMMARY.md`, or the book leaves it out. - Link pages in `docs/` relatively, and files outside it by their full GitHub URL, since the book only has `docs/`. - Draw diagrams as `mermaid` code blocks. -- Say "web client" and "public", and use `publisher.example.com`, `example.com` and `+237600000000` in examples. Keep GitHub alerts for what readers must not miss. -- Start a task page with its requirements. End a page that points onward with a `## Next steps` section of one or two linked sentences or a short list; skip it when the book's next page is the only next step. -- Use tables for reference data, numbered steps for procedures, and one heading per symptom for troubleshooting. -- Write plainly and briefly in US English, one paragraph per line, with sentence-case headings and `` placeholders. - -## When CI fails - -`.github/workflows/checks.yml` runs each check as its own job: **Hooks**, **Tests** on Python 3.12 and 3.13, **Coverage** on 3.14, **SDK** on 3.12 to 3.14, and **SDK version**, which fails when `sdk/src` changed without a new `version` in `sdk/pyproject.toml`. The staging deploy waits only for these. **Database dialects** and **E2E** have their own workflows, which run on PRs that touch their files, or from the Actions tab with **Run workflow**. Every pytest job adds a results table to the run's summary page, through `tests/summary.py`, with the failing tests listed under it. A test module can set `SUMMARY_BY` to group its rows by test or by a parametrized fixture instead of by directory. - -- **Hook failures:** run `make check` locally. Ruff and the whitespace hooks fix files in place, so re-add and commit the changes. -- **Pyright:** fix the type, or narrow it with an `assert`. Use `# pyright: ignore[rule]` only with a reason. -- **import-linter:** a lower layer imports a higher one. Move the code down a layer or pass the dependency in as an argument. -- **Tests pass on newer Pythons but fail on 3.12:** look for newer syntax or an unquoted forward reference. -- **Dialect tests:** run `PYTEST_ARGS="-k mysql" make test-dialects` for the failing dialect. -- **e2e:** run `make test-e2e`, or `PYTEST_ARGS="-k debian" make test-e2e` for the failing distro. -- **Requirements check:** the pinned dependencies conflict. Run `pip install --dry-run -r requirements-observability.txt` to see the conflict. +- Write in US English, one paragraph per line, and match the layout of similar pages. In examples, use `` placeholders, `publisher.example.com`, `example.com` and `+237600000000`. + +## Commits and pull requests + +### Branches + +| Branch | Role | +| --- | --- | +| `staging` | Target of every pull request. Each push deploys to the staging server once the checks pass. | +| `main` | Releases. `staging` is merged into it. | +| Your branch | Named after the change, such as `fix/help-text-typo` or `feat/export-csv`, on your fork. | + +If `staging` moves on while your PR is open, rebase onto it: + +```bash +git fetch upstream +git rebase upstream/staging +git push --force-with-lease +``` + +### Commits + +Messages follow [Conventional Commits](https://www.conventionalcommits.org/), checked by commitizen: + +``` +(): + + + + +``` + +Keep it to the summary line unless the change needs explaining: + +``` +fix(cli): show the missing file when a command fails +feat(reports)!: rename REPORTS_DIR to REPORTS_PATH +``` + +- Types are commitizen's, such as `feat`, `fix`, `refactor`, `test` and `docs`. Scopes name the area, such as `grpc`, `rest` or `auth`. +- Mark breaking changes with `!` and explain the migration path in a `BREAKING CHANGE:` footer. Breaking changes include removed or renamed env vars, CLI commands, endpoints and RPC fields, and anything an operator must do by hand on update. + +### Pull request checklist + +- [ ] The PR targets `staging` and does one thing. Unrelated cleanups go in their own PR. +- [ ] `make check` and `make test` pass locally, plus the [path-triggered checks](#run-the-checks-ci-runs) your change touches. +- [ ] New behavior comes with tests. A bug fix has a test that fails without it. +- [ ] The manual in `docs/` and `template.env` are updated if behavior, settings or commands changed. +- [ ] Schema changes include a migration and a commit message with `[migrate]`. +- [ ] The description says what changed, why, and how it was tested, and links the issue with `Closes #`. +- [ ] Anything operators must do on deploy is called out. + +### Review + +A maintainer reviews every PR. Leave "Allow edits by maintainers" ticked. Answer each comment, push review fixes as new commits rather than amending earlier ones, and re-request review when you're done. Commits are squashed or tidied on merge as needed. ## AI-assisted contributions diff --git a/Dockerfile b/Dockerfile index 48329b5..3638597 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.14.7-slim-bookworm AS builder +FROM python:3.15.0-slim-bookworm AS builder WORKDIR /publisher @@ -19,8 +19,8 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ | sh -s -- -y --no-modify-path ENV PATH="/root/.cargo/bin:${PATH}" -# Outside /publisher, so a source edit doesn't recopy it in the final stage. -# The link puts it where make and scripts/run.sh look for it. +# The venv lives outside /publisher, linked to where make and scripts/run.sh +# look for it. RUN python3 -m venv /venv && ln -s /venv venv COPY requirements.txt requirements-observability.txt ./ COPY sdk ./sdk @@ -29,9 +29,6 @@ RUN --mount=type=cache,sharing=locked,target=/root/.cache/pip \ -r requirements-observability.txt COPY . . -# Submodule URL is SSH-based; rewrite to HTTPS since no SSH key is -# available in the build environment (same fix install.sh applies). -# The cargo caches outlive the layer, so source edits don't redo the Rust build. RUN --mount=type=cache,sharing=locked,target=/root/.cargo/registry \ --mount=type=cache,sharing=locked,target=/root/.cargo/git \ --mount=type=cache,sharing=locked,target=/publisher/lib_relaysms_payload_specs/target \ @@ -39,7 +36,7 @@ RUN --mount=type=cache,sharing=locked,target=/root/.cargo/registry \ && make build -FROM python:3.14.7-slim-bookworm +FROM python:3.15.0-slim-bookworm WORKDIR /publisher diff --git a/Makefile b/Makefile index 108e1c7..d9e0af1 100644 --- a/Makefile +++ b/Makefile @@ -65,11 +65,11 @@ coverage: test-sdk: $(PYTHON) -m pytest -c sdk/pyproject.toml sdk/tests sdk/examples $(PYTEST_ARGS) -## Migrate and query SQLite, SQLCipher, and Postgres, MySQL and MariaDB in podman. +## Migrate and query SQLite and SQLCipher, and Postgres, MySQL and MariaDB in podman. test-dialects: $(PYTHON) -m pytest tests/dialects $(PYTEST_ARGS) -## Install and update in a systemd container with podman (slow, needs network). +## Install, update and uninstall in systemd containers with podman (slow, needs network). test-e2e: $(PYTHON) -m pytest --noconftest tests/e2e $(PYTEST_ARGS) diff --git a/README.md b/README.md index de2e2fc..b9867e5 100644 --- a/README.md +++ b/README.md @@ -15,27 +15,13 @@ curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publishe ## Documentation -The [manual](docs/README.md) starts with: - -- [Quick start](docs/getting-started/quick-start.md) -- [Settings reference](docs/configuration/settings.md) -- [Managing adapters](docs/platforms/manage.md) and the [provider guides](docs/platforms/manage.md#provider-guides) -- [Connecting a relay](docs/gateway-clients/connect-a-relay.md) -- [Troubleshooting](docs/operating/troubleshooting.md) - -`sudo ./manage.sh docs ` serves the manual from your own server. +Read the [manual](docs/README.md). Its [Where to start](docs/README.md#where-to-start) table points to the page for each task. ## Contributing -[CONTRIBUTING.md](CONTRIBUTING.md) covers setup, running the services locally, testing and the workflow. To start: - -```bash -git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git -cd RelaySMS-Publisher -make setup -``` +Read [CONTRIBUTING.md](CONTRIBUTING.md) to set up a development checkout and open a pull request. -Adapters are built with the [adapter SDK](sdk/README.md). +To add a platform, see [Writing an adapter](docs/developers/writing-an-adapter.md). ## Security diff --git a/SECURITY.md b/SECURITY.md index 6334e0e..8b87e70 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,8 +2,8 @@ Don't open a public issue for a vulnerability. Email **** with: -- a description of the issue and its impact; -- the affected versions or commits; -- steps to reproduce it. +- a description of the issue and its impact +- the affected versions or commits +- the steps to reproduce it This covers the Publisher, its adapter SDK in `sdk/` and the platform adapters in the [smswithoutborders](https://github.com/smswithoutborders) organization. diff --git a/alembic.ini b/alembic.ini index 9bacf28..969ba8b 100644 --- a/alembic.ini +++ b/alembic.ini @@ -4,7 +4,7 @@ # path to migration scripts script_location = %(here)s/migrations -# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# template used to generate migration file names. The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time # see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file # for all available tokens diff --git a/deploy/nginx/relaysms-publisher-nginx.conf.template b/deploy/nginx/relaysms-publisher-nginx.conf.template index 9e8bea9..e63f4d4 100644 --- a/deploy/nginx/relaysms-publisher-nginx.conf.template +++ b/deploy/nginx/relaysms-publisher-nginx.conf.template @@ -3,13 +3,12 @@ # Installed to /etc/nginx/sites-available/.conf by install.sh, with # __SERVER_NAME__, __REST_PORT__ and __GRPC_PORT__ substituted from .env. # -# Upstream, map and zone names are global in nginx, so they carry the REST -# port to keep instances on one host apart. +# Upstream, map and zone names end in the REST port, one set per instance on +# a host. # -# Defines only the plain-HTTP (port 80) block. Run `certbot --nginx -d -# ` afterwards (install.sh offers to do this) to add the HTTPS block -# and redirect; certbot copies every directive here into it, so add -# hardening here rather than to the generated 443 block. +# Defines only the port 80 block. `certbot --nginx -d ` adds the HTTPS +# block and redirect, copying every directive here into it. Add hardening here, +# not to the generated 443 block. # Rate-limit logins and requests carrying Basic credentials. limit_req_zone $binary_remote_addr zone=publisher_login___REST_PORT__:10m rate=5r/m; @@ -178,8 +177,8 @@ server { return 494 '{"error": "Request headers are too large."}'; } - # proxy_set_header above doesn't apply to grpc_pass; repeated here via - # grpc_set_header so the gRPC backend also gets the real client IP, etc. + # grpc_pass ignores proxy_set_header: the gRPC locations set the client IP + # headers with grpc_set_header. location /publisher.v3.Publisher { grpc_pass grpc://publisher_grpc___REST_PORT__; grpc_set_header Host $host; diff --git a/deploy/systemd/relaysms-publisher-rest.service b/deploy/systemd/relaysms-publisher-rest.service index 126e92c..ba34b62 100644 --- a/deploy/systemd/relaysms-publisher-rest.service +++ b/deploy/systemd/relaysms-publisher-rest.service @@ -13,8 +13,7 @@ WorkingDirectory=/opt/relaysms/relaysms-publisher EnvironmentFile=/opt/relaysms/relaysms-publisher/.env Environment=OTEL_SERVICE_NAME=relaysms-publisher-rest Environment=PATH=/opt/relaysms/relaysms-publisher/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -# uvicorn keeps replacing workers that fail on startup, so systemd would never see -# the failure. Importing the app once first makes a config error fail the unit. +# Fails the unit on a config error before uvicorn starts its workers. ExecStartPre=/opt/relaysms/relaysms-publisher/venv/bin/python3 -c "import publisher.api.rest.app" ExecStart=/opt/relaysms/relaysms-publisher/scripts/otel-wrap.sh /opt/relaysms/relaysms-publisher/venv/bin/python3 -m uvicorn publisher.api.rest.app:app --host "${HOST}" --port "${PORT}" --workers 4 --proxy-headers --forwarded-allow-ips "*" Restart=on-failure diff --git a/docker-compose.yml b/docker-compose.yml index ac1c7c4..0c55e56 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,9 +7,7 @@ services: container_name: relaysms-publisher volumes: - ./data:/publisher/data - # HOST/GRPC_HOST default to 127.0.0.1 in .env, which would make the - # container unreachable from outside. Override regardless of what .env - # has, since inside a container 0.0.0.0 is the only sensible bind. + # Listens on every interface in the container, whatever .env sets. environment: HOST: 0.0.0.0 GRPC_HOST: 0.0.0.0 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 82db3ad..0397330 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -13,8 +13,5 @@ export WORKERS="${WORKERS:-4}" export GRPC_HOST="${GRPC_HOST:-0.0.0.0}" python3 -m alembic upgrade head -# Registers adapters installed before the registry moved to the database, and -# moves their files out of their code directories. -python3 -m publisher platforms import exec "$SCRIPT_DIR/scripts/run.sh" diff --git a/docs/README.md b/docs/README.md index 39c4401..1620a02 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,11 +5,6 @@ Publishes to online platforms, such as Gmail, from content people send by SMS, s ## How it works ```mermaid ---- -config: - flowchart: - curve: linear ---- flowchart TB app["RelaySMS app"] gateway["Gateway client"] @@ -72,5 +67,5 @@ The [glossary](reference/glossary.md) defines the rest, and [Architecture](expla ## Getting help -- Ask questions and report bugs in the [issue tracker](https://github.com/smswithoutborders/RelaySMS-Publisher/issues); [CONTRIBUTING.md](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/CONTRIBUTING.md#ways-to-contribute) says what a useful report has. +- Ask questions and report bugs in the [issue tracker](https://github.com/smswithoutborders/RelaySMS-Publisher/issues). [CONTRIBUTING.md](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/CONTRIBUTING.md#where-to-start) says what a useful report has. - Report vulnerabilities privately, as [SECURITY.md](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/SECURITY.md) describes. diff --git a/docs/configuration/encryption-keys.md b/docs/configuration/encryption-keys.md index cf99b82..c8305bf 100644 --- a/docs/configuration/encryption-keys.md +++ b/docs/configuration/encryption-keys.md @@ -1,6 +1,6 @@ # Encryption keys -Three keys in `.env` protect the data. The installer and `make .env` generate them. Each is 64 hex characters; `openssl rand -hex 32` makes one, and `manage.sh check` reports a missing or malformed key. +Three keys in `.env` protect the data. The installer and `make .env` generate them. Each is 64 hex characters, which `openssl rand -hex 32` generates. `manage.sh check` reports a missing or malformed key. | Setting | Protects | Used | | --- | --- | --- | diff --git a/docs/configuration/offline-publishing.md b/docs/configuration/offline-publishing.md index cf23460..277f23e 100644 --- a/docs/configuration/offline-publishing.md +++ b/docs/configuration/offline-publishing.md @@ -1,16 +1,18 @@ # Offline publishing -Offline payloads let people send email through [RelaySMS Mail](../platforms/relaysms-mail.md) without linking an account; linked accounts need none of this ([settings](settings.md#offline-publishing)). The app encrypts an offline payload to the server's identity key, and the Publisher publishes it through the `rmail` adapter, which must be installed. Anyone who can reach an allowed protocol can send one. +Offline payloads let people send email through [RelaySMS Mail](../platforms/relaysms-mail.md) without linking an account. Payloads from linked accounts don't depend on these [settings](settings.md#offline-publishing). The app encrypts an offline payload to the server's identity key, and the Publisher publishes it through the `rmail` adapter, which must be installed. Anyone who can reach an allowed protocol can send one. Each payload is tagged with the protocol it arrived on: - `https`: [REST `POST /v1/publications`](../reference/rest-api.md#publishing) - `smtp`: the [SMTP transport](smtp-transport.md) -- An offline payload from a protocol not in `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` is discarded. +## Rules + +- An offline payload from a protocol not in `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` is discarded. An empty list allows every protocol. - An offline payload over `https` must carry `OFFLINE_PUBLISH_SHARED_SECRET` in the request's `tag` field, or it's discarded. - Leaving `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` empty, or listing `https`, without the secret fails `manage.sh check`, and the services refuse to start. -- A discarded payload is logged and recorded as `failed` in the publication stats, with reason `protocol_not_allowed`, `offline_tag_missing` or `offline_tag_invalid`. `POST /v1/publications` still returns success, since the check runs in the queue. +- A discarded payload is logged and recorded as `rejected` in the publication stats, with reason `protocol_not_allowed`, `offline_tag_missing` or `offline_tag_invalid`. `POST /v1/publications` still returns success, since the check runs in the queue. ## Allowing https diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 30a1693..debec4b 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -2,7 +2,7 @@ Settings live in `.env` in the install directory. `template.env` lists each one with its default. -- `sudo ./manage.sh check` reports missing or invalid settings; each service refuses to start on an error in the settings it uses. +- `sudo ./manage.sh check` reports missing or invalid settings. Each service refuses to start on an error in the settings it uses. - Restart the services after a change: `sudo ./manage.sh restart`. - After an update, `sudo ./sync-env.sh` adds the settings from `template.env` that `.env` is missing, and shows the change before writing it. - On Docker, check with `docker compose exec publisher python -m publisher config check` and apply a change with `docker compose up -d`. @@ -17,9 +17,9 @@ Settings live in `.env` in the install directory. `template.env` lists each one | `GRPC_HOST` | `127.0.0.1` | gRPC address. Docker always binds `0.0.0.0`. | | `GRPC_PORT` | `6000` | gRPC port | | `GRPC_MAX_WORKERS` | `10` | gRPC worker threads | -| `GRPC_TLS_ENABLED` | `false` | Serves TLS on `GRPC_PORT` itself; leave off behind nginx ([details](../installation/reverse-proxy.md#grpc-tls-without-nginx)) | +| `GRPC_TLS_ENABLED` | `false` | Serves TLS on `GRPC_PORT` itself. Leave it off behind nginx ([details](../installation/reverse-proxy.md#grpc-tls-without-nginx)) | | `GRPC_TLS_CERT_FILE`, `GRPC_TLS_KEY_FILE` | None | Certificate and key. Required when `GRPC_TLS_ENABLED=true`. | -| `NONCE_TTL_SECONDS` | `600` | How far a gRPC request's timestamp may be from the server's clock; nonces are remembered this long to refuse replays | +| `NONCE_TTL_SECONDS` | `600` | How far a gRPC request's timestamp may be from the server's clock, and how long nonces are remembered to refuse replays | ## Database @@ -29,14 +29,14 @@ See [Database](../installation/database.md) and [Encryption keys](encryption-key | --- | --- | --- | | `DATABASE_DIALECT` | `sqlite` | `sqlite`, `mysql` (MySQL and MariaDB) or `postgres` | | `SQLITE_DATABASE_PATH` | `data/relaysms.db` | SQLite database file | -| `MYSQL_HOST` | `127.0.0.1` | MySQL server | +| `MYSQL_HOST` | `127.0.0.1` (`localhost` if unset) | MySQL server | | `MYSQL_PORT` | `3306` | MySQL port | | `MYSQL_USER`, `MYSQL_PASSWORD` | None | Required for `mysql` | -| `MYSQL_DATABASE` | `relaysms` | Required for `mysql`; created if it doesn't exist | -| `POSTGRES_HOST` | `127.0.0.1` | PostgreSQL server | +| `MYSQL_DATABASE` | `relaysms` | Required for `mysql`. Created if it doesn't exist. | +| `POSTGRES_HOST` | `127.0.0.1` (`localhost` if unset) | PostgreSQL server | | `POSTGRES_PORT` | `5432` | PostgreSQL port | | `POSTGRES_USER`, `POSTGRES_PASSWORD` | None | Required for `postgres` | -| `POSTGRES_DATABASE` | `relaysms` | Required for `postgres`; created if it doesn't exist | +| `POSTGRES_DATABASE` | `relaysms` | Required for `postgres`. Created if it doesn't exist. | | `DATABASE_ENCRYPTION_ENABLED` | `true` (`false` if unset) | Encrypts the SQLite file with SQLCipher. No effect on MySQL or PostgreSQL. | | `DATABASE_ENCRYPTION_KEY` | None | Required for SQLite when encryption is on | | `DATABASE_FIELD_ENCRYPTION_ENABLED` | `true` (`false` if unset) | Encrypts linked accounts' platform tokens | @@ -80,7 +80,7 @@ See [Offline publishing](offline-publishing.md). | Setting | Default | Description | | --- | --- | --- | -| `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` | `smtp` (empty if unset) | Comma-separated protocols whose offline payloads publish: `https`, `smtp`. Empty allows both. | +| `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` | `smtp` (empty if unset) | Comma-separated protocols whose offline payloads publish, such as `smtp,https`. Empty allows every protocol. | | `OFFLINE_PUBLISH_SHARED_SECRET` | None | 64 hex characters. Required when `https` is allowed. | ## SMTP transport @@ -109,7 +109,7 @@ See [Managing adapters](../platforms/manage.md#files). Relative paths are from t | `PLATFORMS_ADAPTERS_VENV_DIR` | `data/platforms/venvs` | Directory for adapters' virtualenvs | | `PLATFORMS_ADAPTERS_CONFIG_DIR` | `data/platforms/config` | Directory for adapters' `credentials.json` files | | `PLATFORMS_ADAPTERS_STATE_DIR` | `data/platforms/state` | Directory for files adapters write, such as databases | -| `PLATFORMS_GITHUB_ORGS` | None | Comma-separated GitHub orgs that administrators may install and update adapters from over the REST API. Empty refuses REST installs; `publisher.sh platforms` is unaffected. | +| `PLATFORMS_GITHUB_ORGS` | None | Comma-separated GitHub orgs that administrators may install and update adapters from over the REST API. Empty refuses REST installs, but not `publisher.sh platforms`. | ## REST API access @@ -136,6 +136,6 @@ See [Observability](../operating/observability.md). | `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | `grpc` or `http/protobuf` | | `OTEL_TRACES_SAMPLER` | `parentbased_traceidratio` | Which traces to keep | | `OTEL_TRACES_SAMPLER_ARG` | `1.0` | Fraction of traces kept, `0` to `1` | -| `OTEL_LOGS_EXPORTER` | `otlp` | `otlp` sends logs to the collector; `none` doesn't | +| `OTEL_LOGS_EXPORTER` | `otlp` | `otlp` sends logs to the collector, and `none` turns log export off | | `OTEL_PYTHON_LOG_CORRELATION` | `true` | Adds trace and span IDs to log records | | `UPTIME_KUMA_WORKER_PUSH_URL`, `UPTIME_KUMA_SMTP_PUSH_URL` | None | Uptime Kuma push URLs for the worker and email listener heartbeats. Blank turns them off. | diff --git a/docs/configuration/smtp-transport.md b/docs/configuration/smtp-transport.md index 4bbf4b2..ca5a6bc 100644 --- a/docs/configuration/smtp-transport.md +++ b/docs/configuration/smtp-transport.md @@ -28,15 +28,13 @@ You should see `Connected to mailbox `. ## How it works 1. Waits for unseen mail in `SMTP_IMAP_MAIL_FOLDER` with IMAP IDLE. -2. Rejects each email unless the sender is allowlisted and authenticated. +2. Drops each email unless the sender is allowlisted and authenticated. 3. Parses the body as JSON and validates the payload. -4. Queues the payload for publication, tagged with protocol `smtp`. Whether offline payloads from email publish depends on [offline publishing](offline-publishing.md). +4. Queues the payload for publication, tagged with protocol `smtp`. 5. Deletes each handled email. An email that hits an unexpected error stays in the folder and is retried on the next cycle. ### Allowlist -`SMTP_ALLOWED_SENDERS` lists exact addresses and domains, comma-separated. Empty allows nothing. - > [!WARNING] > A domain entry trusts every authenticated sender on that domain, since DKIM vouches for the domain, not the mailbox. List an exact address when only one sender should be trusted. diff --git a/docs/developers/reference-client.md b/docs/developers/reference-client.md index cb44553..099e8d9 100644 --- a/docs/developers/reference-client.md +++ b/docs/developers/reference-client.md @@ -26,7 +26,7 @@ python -m tools.client [ARGS] | `exchange-oauth2-code --code ` | Exchanges the code and stores the token | | `revoke-oauth2-token` | Revokes a stored OAuth2 token | | `get-pnba-code [--auth-channel signal]` | Requests a PNBA one-time code | -| `exchange-pnba-code --code [--password ]` | Exchanges the code and stores the token; `--password` for two-step verification | +| `exchange-pnba-code --code [--password ]` | Exchanges the code and stores the token. `--password` is for two-step verification. | | `revoke-pnba-token` | Revokes a stored PNBA token | | `sync-keys` | Replaces the token's 256 client and 256 server ephemeral keys | diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index 67d8ffa..16912e6 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -3,11 +3,6 @@ The Publisher is a set of services sharing one database and one Celery broker. Each platform is reached through an adapter that runs as its own process. ```mermaid ---- -config: - flowchart: - curve: linear ---- flowchart TB app["RelaySMS app"] gateway["Gateway client"] @@ -50,7 +45,7 @@ Every service in the Publisher also reads and writes the database. | gRPC API | Links and unlinks accounts through their adapters, stores their tokens encrypted, and exchanges ephemeral public keys with the app (`SyncKeys`) | | REST API | Takes payloads from gateway clients, and serves the admin endpoints, publication stats and public lists | | Email listener | Reads payloads that gateway clients email to a mailbox | -| Celery worker | Decrypts each queued payload and publishes it through the platform's adapter; also installs and updates adapters | +| Celery worker | Decrypts each queued payload and publishes it through the platform's adapter, and installs and updates adapters | | Celery Beat | Schedules the cleanup jobs and, when set, the worker heartbeat | The database holds linked accounts, keys, credentials, adapters, gateway clients, the audit log and publication stats. The broker carries the queue between the services and the worker; with the default SQLite broker it's two files in `data/`. @@ -65,7 +60,7 @@ Each adapter is a Python package installed from its own repository into its own 2. The REST API or the email listener checks its structure, replaces the sender's address with a keyed hash, and queues it. 3. A multi-part payload waits until all its segments arrive. 4. The worker finds the payload's keys, decrypts it, and calls the platform's adapter to publish it. -5. Only a successful publication uses up the keys; a failure leaves them usable. +5. The keys are used up once the payload decrypts and is ready to send, so a platform's refusal doesn't free them and a payload never publishes twice. A failure before that leaves them usable. 6. The outcome is recorded in the publication stats. The [security model](security-model.md) covers the keys and encryption. diff --git a/docs/explanation/data-retention.md b/docs/explanation/data-retention.md index 8725d91..88c2f92 100644 --- a/docs/explanation/data-retention.md +++ b/docs/explanation/data-retention.md @@ -4,15 +4,15 @@ The cleanup jobs run on the [`CELERY_CLEANUP_CRON` and `CELERY_TOKEN_CLEANUP_CRO | Data | Kept until | | --- | --- | -| Linked accounts' tokens | The user unlinks, or the account has no publication or key sync for `TOKEN_IDLE_MAX_AGE_DAYS` (90); then the Publisher also tries to revoke it on its platform | -| Ephemeral key pools | Each pair is deleted when a publication or token check uses it; the pools go with their token | +| Linked accounts' tokens | The user unlinks, or the account has no publication or key sync for `TOKEN_IDLE_MAX_AGE_DAYS` (90). The Publisher then also tries to revoke it on its platform | +| Ephemeral key pools | Each pair is deleted when a publication or token check uses it. The pools go with their token. | | Incomplete multi-part payloads | All segments arrive, or `PAYLOAD_SESSION_MAX_AGE_HOURS` (3) passes | | Publication stats | Kept | | Audit events and adapter jobs | `AUDIT_RETENTION_DAYS` (365) | | Web sessions | Logout, `AUTH_SESSION_IDLE_MINUTES` (30) idle, or `AUTH_SESSION_MAX_HOURS` (12) | | Credentials, adapters and gateway clients | You remove them | | Server identity keys | Kept | -| Adapters' own files | Up to each adapter; they live in its state directory | +| Adapters' own files | Up to each adapter, in its state directory | | Logs | The system journal's retention, and SigNoz's when `OTEL_EXPORTER_OTLP_ENDPOINT` is set | See the [security model](security-model.md) for what each record holds. diff --git a/docs/explanation/security-model.md b/docs/explanation/security-model.md index 586ec31..990915a 100644 --- a/docs/explanation/security-model.md +++ b/docs/explanation/security-model.md @@ -6,11 +6,11 @@ The RelaySMS protocol is described in the [technical whitepaper](relaysms-whitep | Key | Where | Used for | | --- | --- | --- | -| Server identity keys | 256 X25519 key pairs in the database | Every gRPC request and every payload; the only key for payloads sent without a linked account | -| Ephemeral key pools | Up to 256 client public keys and 256 server key pairs per linked account | One use each, by a payload or a token check; a successful use deletes the pair | +| Server identity keys | 256 X25519 key pairs in the database | Every gRPC request and every payload, and the only key for payloads sent without a linked account | +| Ephemeral key pools | Up to 256 client public keys and 256 server key pairs per linked account | One use each, by a payload or a token check | | [Encryption keys](../configuration/encryption-keys.md) | `.env` | Encrypting the private keys, the platform tokens and, with SQLite, the database file, and keying the sender hash | -The app refills a linked account's pools with the gRPC `SyncKeys` call. A failed publication leaves its key pair usable, so the same SMS can publish when it's sent again. A client must never encrypt different content with a key pair it has already used. +The app refills a linked account's pools with the gRPC `SyncKeys` call. A failed token check, or a payload that fails before sending (for example one that doesn't decrypt), leaves its key pair usable. Once a payload is ready to send, its pair is gone even if the platform then refuses it, so a payload never publishes twice. A client must never encrypt different content with a key pair it has already used. ## What travels encrypted @@ -26,7 +26,7 @@ The app refills a linked account's pools with the gRPC `SyncKeys` call. A failed | Platform tokens | Encrypted with `DATABASE_FIELD_ENCRYPTION_KEY` while `DATABASE_FIELD_ENCRYPTION_ENABLED` is on, as `template.env` sets it | | Private keys | Encrypted with `DATA_ENCRYPTION_KEY` | | Sender phone numbers | Replaced with a keyed hash before anything is stored | -| Message content | Ciphertext while queued, and while a multi-part payload waits for its segments; decrypted only in memory, in the worker and the adapter process it hands it to | +| Message content | Ciphertext while queued, and while a multi-part payload waits for its segments. Decrypted only in memory, in the worker and the adapter process it hands it to | | Publication stats | Platform, protocol, status, country code, failure reason and time | ## Adapters diff --git a/docs/gateway-clients/manage.md b/docs/gateway-clients/manage.md index 8e53fd1..d9a8180 100644 --- a/docs/gateway-clients/manage.md +++ b/docs/gateway-clients/manage.md @@ -1,21 +1,21 @@ # Managing gateway clients -Register each number that relays SMS to this server, so the RelaySMS app lists it, then [connect its relay](connect-a-relay.md). Credentials with `gc:read` and `gc:write` can also manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients); every change is recorded in the [audit log](../operating/audit-log.md). +Register each number that relays SMS to this server, so the RelaySMS app lists it, then [connect its relay](connect-a-relay.md). Credentials with `gc:read` and `gc:write` can also manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients). Every change is recorded in the [audit log](../operating/audit-log.md). ## Commands | Command | Does | | --- | --- | -| `gateway-clients suggest ` | Shows the number's details and PLMN candidates; saves nothing | -| `gateway-clients create --msisdn --protocols ` | Registers a number; also takes `--country`, `--operator` and `--operator-code` | -| `gateway-clients list` | Lists them; filter with `--msisdn`, `--country` or `--operator` | +| `gateway-clients suggest ` | Shows the number's details and PLMN candidates, without saving | +| `gateway-clients create --msisdn --protocols ` | Registers a number. Also takes `--country`, `--operator` and `--operator-code` | +| `gateway-clients list` | Lists them, optionally filtered by `--msisdn`, `--country` or `--operator` | | `gateway-clients update ` | Changes `--country`, `--operator`, `--operator-code` or `--protocols` | | `gateway-clients disable ` | Hides it from the public list | | `gateway-clients enable ` | Lists it publicly again | | `gateway-clients delete ` | Removes it | | `gateway-clients countries` | Lists the countries that have a gateway client | | `gateway-clients operators --country ` | Lists the operators of that country's gateway clients | -| `gateway-clients mcc-mnc list` | Searches the PLMN table; filter with `--country-code`, `--network` or `--iso` | +| `gateway-clients mcc-mnc list` | Searches the PLMN table, optionally filtered by `--country-code`, `--network` or `--iso` | Prefix each with `sudo ./publisher.sh`. `` is how the relay reaches the Publisher, comma-separated: `https` for one that posts to the REST API, `smtp` for one that emails. For example: @@ -23,7 +23,7 @@ Prefix each with `sudo ./publisher.sh`. `` is how the relay reaches t sudo ./publisher.sh gateway-clients create --msisdn +237600000000 --protocols https ``` -`create` fills in any of `--country`, `--operator` and `--operator-code` you leave out with what `suggest` finds. It fills the PLMN code (MCC+MNC) only when every candidate shares it; otherwise run `suggest`, pick a candidate and pass it with `--operator-code`. +`create` fills in any of `--country`, `--operator` and `--operator-code` you leave out with what `suggest` finds. It fills the PLMN code (MCC+MNC) only when every candidate shares it. Otherwise, run `suggest`, pick a candidate and pass it with `--operator-code`. ## How suggest works diff --git a/docs/installation/database.md b/docs/installation/database.md index 4b272df..99b4fed 100644 --- a/docs/installation/database.md +++ b/docs/installation/database.md @@ -6,7 +6,7 @@ | MySQL 8.0.28+ or MariaDB | Heavier load, or a database on another host | The server's own (TDE) | | PostgreSQL 12+ | Heavier load, or a database on another host | The disk's | -In every dialect, the server's private keys are encrypted, and so are platform tokens while field encryption is on; see [Encryption keys](../configuration/encryption-keys.md). +In every dialect, the server's private keys are encrypted, and so are platform tokens while field encryption is on. See [Encryption keys](../configuration/encryption-keys.md). ## Database servers @@ -28,7 +28,7 @@ For MySQL or MariaDB, use `--setup-db mysql` or `scripts/setup-mysql.sh` the sam `--db-password ` sets the password instead of generating one. -To use a server you already have, pass its details. The script only checks the connection and writes `.env`; it never changes the server: +To use a server you already have, pass its details. The script only checks the connection and writes `.env`, and never changes the server: ```bash sudo ./install.sh --setup-db postgres --db-existing \ diff --git a/docs/installation/docker.md b/docs/installation/docker.md index d7b16ee..7189a66 100644 --- a/docs/installation/docker.md +++ b/docs/installation/docker.md @@ -13,8 +13,6 @@ The container applies pending migrations, then starts every service. It publishe Run the admin CLI with `docker compose exec publisher python -m publisher `. -[Back up](../operating/backup-restore.md) `.env` and `./data`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. - To update, see [Upgrading](../operating/upgrading.md#update-with-docker). ## Next steps diff --git a/docs/installation/multiple-instances.md b/docs/installation/multiple-instances.md index d1829bd..569c2b3 100644 --- a/docs/installation/multiple-instances.md +++ b/docs/installation/multiple-instances.md @@ -18,4 +18,4 @@ To run a second Publisher on the same host, give it its own install directory, i sudo ./manage.sh nginx acme.example.com ``` -Its units are named after the instance (`relaysms-publisher-acme.target`, `relaysms-publisher-acme-rest.service`, ...), and the `manage.sh` in each install directory manages only its own instance. Running `install.sh` again on that directory reuses the name; a different `--instance-name` is refused. +Its units are named after the instance (`relaysms-publisher-acme.target`, `relaysms-publisher-acme-rest.service`, ...), and the `manage.sh` in each install directory manages only its own instance. Running `install.sh` again on that directory reuses the name, and refuses a different `--instance-name`. diff --git a/docs/installation/requirements.md b/docs/installation/requirements.md index d9ed2c4..83d9907 100644 --- a/docs/installation/requirements.md +++ b/docs/installation/requirements.md @@ -5,6 +5,6 @@ - **Database**: SQLite (the default) or a database server. See [Database](database.md) for the supported ones. - **Domain**: a DNS name pointing at the server, for the TLS certificate. See [Reverse proxy and TLS](reverse-proxy.md). - **Inbound ports (firewall)**: 80 and 443 for nginx. On a server install, the services listen on `127.0.0.1` only. Docker publishes `PORT` (16000) and `GRPC_PORT` (6000). -- **Outbound access**: `github.com`, `pypi.org`, the Rust installer and the system's package mirrors to install the Publisher and its adapters; Let's Encrypt for certificates; each platform's API to publish. +- **Outbound access**: `github.com`, `pypi.org`, the Rust installer and the system's package mirrors to install the Publisher and its adapters, `cdn.jsdelivr.net` to build the manual, Let's Encrypt for certificates, and each platform's API to publish. - **Platform credentials**: from each platform you offer, such as a Google Cloud OAuth client for Gmail. See [Provider guides](../platforms/manage.md#provider-guides). - **Gateway client**: at least one phone number that relays SMS, such as an Android phone running Deku SMS. See [Connecting a relay](../gateway-clients/connect-a-relay.md). diff --git a/docs/installation/reverse-proxy.md b/docs/installation/reverse-proxy.md index 32798be..46b8e18 100644 --- a/docs/installation/reverse-proxy.md +++ b/docs/installation/reverse-proxy.md @@ -4,7 +4,7 @@ The services listen on `127.0.0.1`. nginx serves them on port 443 with a Let's E ## With the installer -Pass `--site-name ` to the [installer](server.md#run-the-installer). Running it again leaves an existing site file alone, and skips Certbot when the domain already has a certificate; use [`manage.sh nginx`](#on-an-existing-install) to render the site again. +Pass `--site-name ` to the [installer](server.md#run-the-installer). Running it again leaves an existing site file alone, and skips Certbot when the domain already has a certificate. Use [`manage.sh nginx`](#on-an-existing-install) to render the site again. ## On an existing install diff --git a/docs/installation/server.md b/docs/installation/server.md index 1fe26ea..36cbdb1 100644 --- a/docs/installation/server.md +++ b/docs/installation/server.md @@ -50,7 +50,7 @@ The installer: 5. installs and starts the systemd services; 6. sets up nginx and the certificate, if you gave a domain. -Running it again updates the install in place and keeps its `.env`. Run it as the same user as the first time; it refuses another service user. +Running it again updates the install in place and keeps its `.env`. Run it as the same user as the first time, since it refuses another service user. ## After installing diff --git a/docs/openapi.json b/docs/openapi.json index b659ca3..9c0e32d 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -11,7 +11,7 @@ "Auth" ], "summary": "Log in", - "description": "Sets an HttpOnly session cookie for the web client; scripts use HTTP Basic.", + "description": "Sets an HttpOnly session cookie for the web client. Scripts use HTTP Basic.", "operationId": "login_v1_auth_login_post", "requestBody": { "content": { @@ -55,7 +55,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -286,7 +286,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -384,7 +384,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -508,7 +508,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -615,7 +615,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -643,7 +643,7 @@ "Credentials" ], "summary": "Reset password", - "description": "New password shown once; sessions end. Needs If-Match. Scope: creds:write.", + "description": "New password shown once, and sessions end. Needs If-Match. Scope: creds:write.", "operationId": "reset_credential_password_v1_creds__username__reset_password_post", "security": [ { @@ -731,7 +731,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -759,7 +759,7 @@ "Credentials" ], "summary": "Revoke sessions", - "description": "Logs it out everywhere; Basic auth still works. Scope: creds:write.", + "description": "Logs it out everywhere. Basic auth still works. Scope: creds:write.", "operationId": "revoke_credential_sessions_v1_creds__username__revoke_sessions_post", "security": [ { @@ -830,7 +830,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1033,7 +1033,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1260,7 +1260,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1278,7 +1278,7 @@ "Stats" ], "summary": "Summarize publications", - "description": "Counts per group; last 30 days by default. Scope: stats:publications:read.\n\nGrouping by `failure_reason` also needs stats:publications:reasons. Groups sort by\nperiod, then count, highest first. Periods with no rows are left out; chart them\nas 0.", + "description": "Counts per group (last 30 days by default). Scope: stats:publications:read.\n\nGrouping by `failure_reason` also needs stats:publications:reasons. Groups sort by\nperiod, then count, highest first. Periods with no rows are left out.", "operationId": "summarize_publication_stats_v1_stats_publications_summary_get", "security": [ { @@ -1489,7 +1489,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1507,7 +1507,7 @@ "Publishing" ], "summary": "Publish", - "description": "Queues an SMS payload for publishing.\n\nMulti-part payloads are kept until every segment arrives, then published. Payloads\nare tagged with protocol `https`; see Offline publishing in the manual for\n`OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors\nhappen later in the queue, so they're logged, not returned.", + "description": "Queues an SMS payload for publishing.\n\nMulti-part payloads are kept until every segment arrives, then published. Payloads\nare tagged with protocol `https`. See Offline publishing in the manual for\n`OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors are\nlogged, not returned.", "operationId": "create_publications_v1_publications_post", "requestBody": { "content": { @@ -1541,7 +1541,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1635,7 +1635,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1691,7 +1691,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1743,7 +1743,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1817,7 +1817,7 @@ "Platform Adapters" ], "summary": "Install adapter", - "description": "Queues an install; poll the job at Location. Administrators only.", + "description": "Queues an install. Poll the job at Location. Administrators only.", "operationId": "install_adapter_v1_platforms_adapters_post", "requestBody": { "content": { @@ -1881,7 +1881,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -1990,7 +1990,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -2105,7 +2105,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -2131,7 +2131,7 @@ "Platform Adapters" ], "summary": "Uninstall adapter", - "description": "Needs If-Match. Scope: platforms:write.\n\nRefused while accounts are linked through it; disable it instead.", + "description": "Needs If-Match. Scope: platforms:write.\n\nRefused while accounts are linked through it. Disable it instead.", "operationId": "delete_adapter_v1_platforms_adapters__adapter_id__delete", "security": [ { @@ -2213,7 +2213,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -2310,7 +2310,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -2445,7 +2445,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -2592,7 +2592,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -2630,7 +2630,7 @@ "Platform Adapters" ], "summary": "Update adapter", - "description": "Queues a move to a version tag; poll the job at Location. Administrators only.", + "description": "Queues a move to a version tag. Poll the job at Location. Administrators only.", "operationId": "update_adapter_version_v1_platforms_adapters__adapter_id__update_post", "security": [ { @@ -2729,7 +2729,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -2825,7 +2825,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -2906,7 +2906,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -3001,7 +3001,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -3139,7 +3139,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -3239,7 +3239,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -3325,7 +3325,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -3449,7 +3449,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -3556,7 +3556,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -3648,7 +3648,7 @@ } }, "422": { - "description": "A parameter or body field is missing or invalid; the message names it.", + "description": "A parameter or body field is missing or invalid. The message names it.", "content": { "application/json": { "schema": { @@ -3708,7 +3708,7 @@ }, "type": "array", "title": "Setup", - "description": "Inputs POST .../setup takes; empty if it has no setup." + "description": "Inputs POST .../setup takes, or empty if it has no setup." } }, "type": "object", @@ -3724,7 +3724,7 @@ "additionalProperties": true, "type": "object", "title": "Settings", - "description": "Settings to change; others are kept, and null removes one." + "description": "Settings to change. Others are kept, and null removes one." } }, "additionalProperties": false, @@ -4024,7 +4024,7 @@ } ], "title": "Actor", - "description": "Username at the time; null for the CLI or a failed login." + "description": "Username at the time, or null for the CLI or a failed login." }, "action": { "type": "string", @@ -4082,7 +4082,7 @@ } ], "title": "Next", - "description": "URL of the next (older) page; null on the last page." + "description": "URL of the next (older) page, or null on the last page." }, "prev": { "anyOf": [ @@ -4094,7 +4094,7 @@ } ], "title": "Prev", - "description": "URL of the previous (newer) page; null on the first page." + "description": "URL of the previous (newer) page, or null on the first page." }, "data": { "items": { @@ -4499,7 +4499,7 @@ } ], "title": "Created By", - "description": "Username; null for the CLI or a deleted credential." + "description": "Username, or null for the CLI or a deleted credential." }, "updated_at": { "type": "string", @@ -4516,7 +4516,7 @@ } ], "title": "Updated By", - "description": "Username; null for the CLI or a deleted credential." + "description": "Username, or null for the CLI or a deleted credential." } }, "type": "object", @@ -4926,7 +4926,7 @@ } ], "title": "Tag", - "description": "Null for an adapter registered by `platforms import`." + "description": "Null for an adapter registered before tags were recorded, until its first update." }, "commit": { "type": "string", @@ -4956,7 +4956,7 @@ } ], "title": "Created By", - "description": "Username; null for the CLI or a deleted credential." + "description": "Username, or null for the CLI or a deleted credential." }, "updated_at": { "type": "string", @@ -4973,7 +4973,7 @@ } ], "title": "Updated By", - "description": "Username; null for the CLI or a deleted credential." + "description": "Username, or null for the CLI or a deleted credential." } }, "type": "object", @@ -5110,7 +5110,8 @@ }, "status": { "type": "string", - "title": "Status" + "title": "Status", + "description": "published, failed or rejected (refused by the offline publishing settings)." }, "country_code": { "anyOf": [ @@ -5133,7 +5134,7 @@ } ], "title": "Failure Reason", - "description": "Why it failed, as a short code such as payload_malformed or adapter_rate_limited." + "description": "Why it failed or was rejected, as a short code such as payload_malformed or offline_tag_invalid." }, "created_at": { "type": "string", @@ -5161,7 +5162,7 @@ } ], "title": "Next", - "description": "URL of the next (older) page; null on the last page." + "description": "URL of the next (older) page, or null on the last page." }, "prev": { "anyOf": [ @@ -5173,7 +5174,7 @@ } ], "title": "Prev", - "description": "URL of the previous (newer) page; null on the first page." + "description": "URL of the previous (newer) page, or null on the first page." }, "data": { "items": { @@ -5270,7 +5271,7 @@ } ], "title": "Error", - "description": "Always null here; errors come back with a 4xx status." + "description": "Always null here. Errors come back with an error status." } }, "type": "object", diff --git a/docs/operating/audit-log.md b/docs/operating/audit-log.md index d41db03..d6761e3 100644 --- a/docs/operating/audit-log.md +++ b/docs/operating/audit-log.md @@ -12,7 +12,7 @@ It needs `audit:read`, plus the read scope of each area: `creds:read` for `auth. | Action | Recorded when | | --- | --- | -| `creds.create`, `creds.update`, `creds.reset_password`, `creds.revoke_sessions`, `creds.delete` | A credential changes. `details` says what changed, never secrets. | +| `creds.create`, `creds.update`, `creds.reset_password`, `creds.revoke_sessions`, `creds.delete` | A credential changes. `details` says what changed. | | `auth.login` | A web session starts, or a web or HTTP Basic login for an existing username is refused | | `auth.logout` | A web session ends | | `platforms.add`, `platforms.update`, `platforms.remove` | An adapter is installed, updated or removed. `details` has the source URL and commits. | @@ -24,7 +24,7 @@ It needs `audit:read`, plus the read scope of each area: `creds:read` for `auth. ## Fields - `outcome`: `success`, `denied` (the change went beyond the actor's scopes) or `failed` (a refused login for an existing username: wrong password, or disabled). -- `actor`: the username at the time; null for the CLI or a failed login. -- `details`: what changed, never secrets; for `auth.login`, `method` is `session` or `basic`. +- `actor`: the username at the time, or null for the CLI or a failed login. +- `details`: what changed. For `auth.login`, `method` is `session` or `basic`. Successful HTTP Basic requests aren't recorded. Events are kept for `AUDIT_RETENTION_DAYS` ([Retention](../configuration/settings.md#retention)). diff --git a/docs/operating/backup-restore.md b/docs/operating/backup-restore.md index 0089f0f..ee6e07f 100644 --- a/docs/operating/backup-restore.md +++ b/docs/operating/backup-restore.md @@ -46,7 +46,7 @@ For a database on another host, or named differently in `.env`, connect with the ## Restore -1. [Install](../installation/server.md) the Publisher in the same directory, with the same `--instance-name`. With MySQL or PostgreSQL, pass `--setup-db`, `--db-name`, `--db-user` and `--db-password` with the values from the backed-up `.env` (`MYSQL_*` or `POSTGRES_*`); with RabbitMQ, the same for `--broker-password`. Otherwise the servers refuse the restored `.env`'s logins. +1. [Install](../installation/server.md) the Publisher in the same directory, with the same `--instance-name`. With MySQL or PostgreSQL, pass `--setup-db`, `--db-name`, `--db-user` and `--db-password` with the values from the backed-up `.env` (`MYSQL_*` or `POSTGRES_*`). With RabbitMQ, do the same for `--broker-password`. Otherwise the servers refuse the restored `.env`'s logins. 2. Stop the services: ```bash diff --git a/docs/operating/credentials.md b/docs/operating/credentials.md index abe4d2a..3d30490 100644 --- a/docs/operating/credentials.md +++ b/docs/operating/credentials.md @@ -35,12 +35,12 @@ Usernames are 3 to 32 lowercase letters, digits, `.`, `_` or `-`, starting with | Command | Does | | --- | --- | | `creds list` | Lists credentials | -| `creds set-scopes --username --scope ` | Replaces a credential's scopes; repeat `--scope`, or pass `--administrator` | +| `creds set-scopes --username --scope ` | Replaces a credential's scopes. Repeat `--scope`, or pass `--administrator` | | `creds reset-password --username ` | Prints a new password and ends the credential's web sessions | | `creds disable --username ` | Blocks it and ends its web sessions | | `creds enable --username ` | Lets it log in again | -| `creds revoke-sessions --username ` | Logs it out of every web session; HTTP Basic still works | -| `creds delete --username ` | Deletes it and its sessions, after asking; `--yes` skips the question | +| `creds revoke-sessions --username ` | Logs it out of every web session. HTTP Basic still works. | +| `creds delete --username ` | Deletes it and its sessions, after asking. `--yes` skips the question. | Prefix each with `sudo ./publisher.sh`. Credentials with `creds:write` can do the same over the [REST API](../reference/rest-api.md#managing-credentials), within their own scopes. diff --git a/docs/operating/observability.md b/docs/operating/observability.md index fa97842..a0d73f6 100644 --- a/docs/operating/observability.md +++ b/docs/operating/observability.md @@ -152,7 +152,7 @@ curl -X POST http://127.0.0.1:8090/api/v1/register \ -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@example.com","password":""}' ``` -The template stores SigNoz's metadata in Postgres (`metastore.kind: postgres`); set `sqlite` for a single node. +The template stores SigNoz's metadata in Postgres (`metastore.kind: postgres`). Set `sqlite` for a single node. ### Uptime Kuma diff --git a/docs/operating/security-hardening.md b/docs/operating/security-hardening.md index b83a69e..0e946a9 100644 --- a/docs/operating/security-hardening.md +++ b/docs/operating/security-hardening.md @@ -27,7 +27,7 @@ The services run as the user who ran the installer with `sudo`, or as a new `rel | `AUTH_SESSION_COOKIE_SECURE` | `true` | | `AUTH_WEB_ORIGINS` | Only the web client origins you run | | `PLATFORMS_GITHUB_ORGS` | Only orgs you trust: installing an adapter runs its code on the server | -| `SMTP_ALLOWED_SENDERS` | Exact addresses; a domain trusts every sender on it ([details](../configuration/smtp-transport.md#allowlist)) | +| `SMTP_ALLOWED_SENDERS` | Exact addresses, since a domain trusts every sender on it ([details](../configuration/smtp-transport.md#allowlist)) | | `DATABASE_ENCRYPTION_ENABLED`, `DATABASE_FIELD_ENCRYPTION_ENABLED` | `true` | ## Credentials diff --git a/docs/operating/services.md b/docs/operating/services.md index 9799aea..138599d 100644 --- a/docs/operating/services.md +++ b/docs/operating/services.md @@ -8,7 +8,7 @@ The Publisher runs as systemd services, grouped under `relaysms-publisher.target | `relaysms-publisher-grpc` | gRPC API | | `relaysms-publisher-worker` | Celery worker: publishes queued payloads | | `relaysms-publisher-beat` | Celery Beat: schedules cleanups, and the worker heartbeat with Uptime Kuma | -| `relaysms-publisher-smtp` | [Email listener](../configuration/smtp-transport.md); exits at once while `SMTP_TRANSPORT_ENABLED` is off | +| `relaysms-publisher-smtp` | [Email listener](../configuration/smtp-transport.md). Exits at once while `SMTP_TRANSPORT_ENABLED` is off | The REST API listens on `HOST:PORT` (`127.0.0.1:16000`) and gRPC on `GRPC_HOST:GRPC_PORT` (`127.0.0.1:6000`). @@ -25,7 +25,7 @@ You should see every service active, and `{"status":"ok"}`. To follow publicatio sudo ./manage.sh logs --unit worker ``` -Each published payload logs `Published via ''.`, and each failure a line starting `Failed to` ([what each means](troubleshooting.md#a-payload-is-accepted-but-never-published)). Credentials with `stats:publications:read` can also query the counts at `GET /v1/stats/publications`. +The worker logs each payload's outcome ([what each line means](troubleshooting.md#a-payload-is-accepted-but-never-published)). Credentials with `stats:publications:read` can also query the counts at `GET /v1/stats/publications`. ## manage.sh @@ -48,7 +48,7 @@ Run it from the install directory with `sudo`: `logs` takes: -- `-u`, `--unit`: `rest`, `grpc`, `worker`, `beat` or `smtp`; repeatable +- `-u`, `--unit`: `rest`, `grpc`, `worker`, `beat` or `smtp`, repeatable - `-n`, `--lines`: how many lines to show - `-s`, `--since`: a time, such as `"1 hour ago"` - `--no-follow`: print and exit diff --git a/docs/operating/troubleshooting.md b/docs/operating/troubleshooting.md index e3ac2ea..daa5577 100644 --- a/docs/operating/troubleshooting.md +++ b/docs/operating/troubleshooting.md @@ -39,13 +39,15 @@ The container applies migrations before the services start. Wait, or follow `doc | Log line | Cause | | --- | --- | | `Published via ''.` | It worked | -| `Failed to process payload: ...` | The payload didn't decrypt or validate, its keys are unknown or used up, or offline publishing refused it ([Offline publishing](../configuration/offline-publishing.md)) | -| `Failed to publish message: Adapter error: ...` | The platform refused it; the rest of the line is the adapter's reason. Fix it with the platform's [provider guide](../platforms/manage.md#provider-guides), then the user sends the same SMS again. | +| `Failed to process payload: ...` | The payload didn't decrypt or validate, or its keys are unknown or used up | +| `Rejected payload: ...` | Offline publishing refused it ([Offline publishing](../configuration/offline-publishing.md)) | +| `Failed to publish message: Adapter error: ...` | The platform refused it. The rest of the line is the adapter's reason. Fix it with the platform's [provider guide](../platforms/manage.md#provider-guides), then the user sends the message again. | +| `Sent via '', but couldn't record the result.` | It was published, but a database error kept it out of the stats. The traceback follows. | | `Session can't be assembled from segment(s): ...` | A multi-part payload's segments don't fit together | -| `An unexpected error occurred during task processing.` | A bug; the traceback follows. [Report it](https://github.com/smswithoutborders/RelaySMS-Publisher/issues). | -| No line at all | The worker isn't running (`sudo ./manage.sh status`), or a multi-part payload is still waiting for its other segments, for up to `PAYLOAD_SESSION_MAX_AGE_HOURS` | +| `An unexpected error occurred during task processing.` | A bug. The traceback follows. [Report it](https://github.com/smswithoutborders/RelaySMS-Publisher/issues). | +| No line at all | The worker isn't running (`sudo ./manage.sh status`), or a multi-part payload is still waiting for its other segments, for up to `PAYLOAD_SESSION_MAX_AGE_HOURS`, then it's discarded and recorded as `failed` with reason `segments_missing` | -`GET /v1/stats/publications` lists each failure with a reason code, for credentials with `stats:publications:reasons`. +`GET /v1/stats/publications` lists each failure and rejection with a reason code, for credentials with `stats:publications:reasons`. ### Linking an account fails @@ -66,7 +68,7 @@ The webhook URL in the Twilio console doesn't match the URL the relay sees, sche ### The Twilio relay answers `502` `Forwarding failed.` -A `FORWARD_URLS_JSON` URL refused the payload or couldn't be reached; the relay's log names it. Check that `https:///v1/publications` answers from the relay's host. +A `FORWARD_URLS_JSON` URL refused the payload or couldn't be reached. The relay's log names it. Check that `https:///v1/publications` answers from the relay's host. ## REST API @@ -84,7 +86,7 @@ The session idled past `AUTH_SESSION_IDLE_MINUTES`, reached `AUTH_SESSION_MAX_HO ### `Origin not allowed.` -The web client's origin is neither the API's own nor listed in `AUTH_WEB_ORIGINS`; logins and writes with a session cookie both check it. Add the origin, which must be on the same site as the API, then `sudo ./manage.sh restart`. +The web client's origin is neither the API's own nor listed in `AUTH_WEB_ORIGINS`. Logins and writes with a session cookie both check it. Add the origin, which must be on the same site as the API, then `sudo ./manage.sh restart`. ### `Too many requests. Please try again shortly.` diff --git a/docs/operating/upgrading.md b/docs/operating/upgrading.md index 4bef4aa..19534bd 100644 --- a/docs/operating/upgrading.md +++ b/docs/operating/upgrading.md @@ -24,7 +24,7 @@ sudo ./manage.sh restart 1. `update --migrate` stops the services, pulls the code, reinstalls the dependencies, rebuilds, applies pending migrations and starts the services again. When the install serves this manual, it rebuilds it too. 2. `sync-env.sh` adds the settings the new `template.env` has and `.env` lacks, showing the change before writing it. -3. `check` reports any setting still missing or invalid; fill it in `.env`. +3. `check` reports any setting still missing or invalid. Fill it in `.env`. 4. `restart` loads the new settings. When `.env` has an invalid setting, `update` still restarts every service, then exits with an error ([fix](troubleshooting.md#update-ends-with-env-has-errors)). @@ -40,7 +40,7 @@ If you edited tracked files that the update changes, `update` refuses to pull. D ## Roll back 1. Take the commit you noted [before upgrading](#before-you-upgrade). -2. If the update applied migrations, [restore the backup](backup-restore.md#restore) taken before it; older code can't use the migrated database. +2. If the update applied migrations, [restore the backup](backup-restore.md#restore) taken before it. 3. Check out that commit and rebuild. `--pulled` skips the pull that would undo the checkout: ```bash @@ -57,7 +57,7 @@ git pull --recurse-submodules docker compose up -d --build ``` -The container applies new migrations and runs `platforms import` when it starts. +The container applies new migrations when it starts. ## Update adapters @@ -68,19 +68,3 @@ sudo ./publisher.sh platforms update ``` See [Versions](../platforms/manage.md#versions). - -## Upgrade notes - -Steps a particular upgrade needs once, newest first. - -### Gateway client registry import (migration 019) - -For installs from before 2026-10-07. - -`update --migrate` applies migration 019, which imports `data/gateway_clients/registry.json` (or `GATEWAY_CLIENTS_REGISTRY_FILE`, if `.env` sets it). It logs the entries in `mcc_mnc_overrides.json` without importing them. Delete both files afterwards. - -### Adapter registry import - -For installs from before 2026-10-06. - -`update --migrate` and `manage.sh migrate` register adapter directories on disk that the database lacks, and move their settings and files out of the code. The Docker container does it at every start. diff --git a/docs/platforms/manage.md b/docs/platforms/manage.md index eda6e29..a0c4bed 100644 --- a/docs/platforms/manage.md +++ b/docs/platforms/manage.md @@ -2,8 +2,6 @@ Each platform, such as Gmail, is served by an adapter installed from its own GitHub repository. -Run these commands from the install directory with [`publisher.sh`](../operating/services.md#publishersh), which runs them as the service user. - ## Commands | Command | Does | @@ -11,16 +9,15 @@ Run these commands from the install directory with [`publisher.sh`](../operating | `platforms add [--tag ]` | Installs the newest version tag, or `` | | `platforms update [] [--tag ]` | Moves one adapter, or all, to a version tag | | `platforms list` | Lists every adapter, disabled ones included | -| `platforms disable ` | Hides it from users; linked accounts can still be unlinked | +| `platforms disable ` | Hides it from users. Linked accounts can still be unlinked. | | `platforms enable ` | Offers it again | | `platforms remove [--force]` | Uninstalls it ([details](#removing)) | | `platforms config show ` | Lists its settings, without secret values | -| `platforms config set =\|` | Changes settings; a bare `` asks for its value | +| `platforms config set =\|` | Changes settings. A bare `` asks for its value, and `--unset ` removes one. | | `platforms setup [--input =]` | Runs the adapter's setup and saves its settings | -| `platforms import` | Registers adapter directories on disk that the database lacks, and moves their files out of the code ([details](../operating/upgrading.md#adapter-registry-import)) | | `platforms exec -- ` | Runs a command the adapter installs | -Prefix each with `sudo ./publisher.sh`, for example: +Prefix each with [`sudo ./publisher.sh`](../operating/services.md#publishersh), from the install directory, for example: ```bash sudo ./publisher.sh platforms add https://github.com/smswithoutborders/gmail-oauth2-adapter @@ -70,7 +67,7 @@ sudo ./publisher.sh platforms setup gmail --input client_file=@/tmp/client_secre ## Redirect URIs -OAuth2 adapters, such as Gmail, send the user back to a redirect URI after they authorize. The client app chooses it and sends it to the Publisher when it links an account; the adapter uses the first of its `redirect_uris` setting when the app sends none. +OAuth2 adapters, such as Gmail, send the user back to a redirect URI after they authorize. The client app chooses it and sends it to the Publisher when it links an account. The adapter uses the first of its `redirect_uris` setting when the app sends none. 1. Run an HTTPS page that receives the authorization `code` and `state` and hands them back to the app. 2. Configure the apps to send that page's URI. @@ -89,7 +86,7 @@ OAuth2 adapters, such as Gmail, send the user back to a redirect URI after they ## An adapter's own commands -An adapter may install admin commands. `exec` runs one from the adapter's virtualenv with its config and state directories; put `--` before it: +An adapter may install admin commands. `exec` runs one from the adapter's virtualenv with its config and state directories. Put `--` before it: ```bash sudo ./publisher.sh platforms exec -- --help diff --git a/docs/platforms/mastodon.md b/docs/platforms/mastodon.md index c854e0c..1e909e1 100644 --- a/docs/platforms/mastodon.md +++ b/docs/platforms/mastodon.md @@ -30,7 +30,7 @@ sudo ./publisher.sh platforms setup mastodon \ | `website` | No | Home page on the consent page | | `base_url` | No | The server. Default: `https://mastodon.social` | -`setup` registers the app with the scopes `profile`, `write:statuses` and `write:media`, and saves its client ID and secret. Run it again to change the redirect URIs or the server; it registers a new app and replaces every setting. After changing the server, users must link again. +`setup` registers the app with the scopes `profile`, `write:statuses` and `write:media`, and saves its client ID and secret. Run it again to change the redirect URIs or the server. It registers a new app and replaces every setting. After changing the server, users must link again. ## Check it diff --git a/docs/platforms/relaysms-mail.md b/docs/platforms/relaysms-mail.md index 273b4c2..e0efb13 100644 --- a/docs/platforms/relaysms-mail.md +++ b/docs/platforms/relaysms-mail.md @@ -13,9 +13,9 @@ The `rmail` adapter gives each user an email alias tied to their phone number, a ## SimpleLogin -1. Sign up at [app.simplelogin.io](https://app.simplelogin.io). The signup address becomes your mailbox; to use another, add and verify it under [Mailboxes](https://app.simplelogin.io/dashboard/mailbox). +1. Sign up at [app.simplelogin.io](https://app.simplelogin.io). The signup address becomes your mailbox. To use another, add and verify it under [Mailboxes](https://app.simplelogin.io/dashboard/mailbox). 2. Add your domain, or a subdomain such as `mail.example.com`, under [Custom domains](https://app.simplelogin.io/dashboard/custom_domain). -3. Create the DNS records the domain's page lists: ownership TXT, MX, SPF, DKIM and DMARC. SimpleLogin marks the last three optional; without them, mail sent from aliases doesn't arrive. +3. Create the DNS records the domain's page lists: ownership TXT, MX, SPF, DKIM and DMARC. SimpleLogin marks the last three optional, but without them mail sent from aliases doesn't arrive. 4. Create an [API key](https://app.simplelogin.io/dashboard/api_key). ## SMTP diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 020fe3c..1bb592b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -10,7 +10,7 @@ | `scripts/setup-rabbitmq.sh` | Sets up a RabbitMQ broker | [Celery broker](../installation/celery-broker.md) | | `scripts/setup-observability.sh` | Sets up SigNoz and Uptime Kuma | [Observability](../operating/observability.md) | -Run them with `sudo`; all but `install.sh` from the install directory. Each takes `--help`. +Run them with `sudo`, and all but `install.sh` from the install directory. Each takes `--help`. ## manage.sh @@ -23,9 +23,9 @@ Runs `python3 -m publisher` as the service user. `sudo ./publisher.sh /v1 ## Authentication -Admin endpoints take a web session cookie or HTTP Basic credentials; see [Logging in](../operating/credentials.md#logging-in). Each endpoint's description names the scopes it needs. +Admin endpoints take a web session cookie or HTTP Basic credentials ([Logging in](../operating/credentials.md#logging-in)). Each endpoint's description names the scopes it needs. ## Managing credentials @@ -24,15 +24,15 @@ A credential can only grant scopes it holds, can't change itself, and can't chan ## Managing platform adapters -`/v1/platforms/adapters` lists every installed adapter, with its source, version tag, commit and who last changed it, and takes the same `ETag` and `If-Match` rules as credentials. A disabled adapter disappears from `/v1/platforms` and can't publish or link accounts, but still revokes tokens so users can unlink. Uninstalling is refused with `409` while accounts are linked through the adapter; see [Removing](../platforms/manage.md#removing). +`/v1/platforms/adapters` lists every installed adapter, with its source, version tag, commit and who last changed it, and takes the same `ETag` and `If-Match` rules as credentials. A disabled adapter disappears from `/v1/platforms` and can't publish or link accounts, but still revokes tokens so users can unlink. Uninstalling is refused with `409` while accounts are linked through the adapter. See [Removing](../platforms/manage.md#removing). -Installing (`POST /v1/platforms/adapters`) and updating (`POST /v1/platforms/adapters/{id}/update`) run code from the repository on the server, so they need an administrator and a repository from an org in `PLATFORMS_GITHUB_ORGS`. Both take an optional `tag`, defaulting to the newest version tag, and return `202` with the job's URL in `Location`. Poll the job until its `state` is `succeeded` or `failed`; its `log` has the end of the git and pip output. An adapter has one job at a time; another gets `409`. +Installing (`POST /v1/platforms/adapters`) and updating (`POST /v1/platforms/adapters/{id}/update`) run code from the repository on the server, so they need an administrator and a repository from an org in `PLATFORMS_GITHUB_ORGS`. Both take an optional `tag`, defaulting to the newest version tag, and return `202` with the job's URL in `Location`. Poll the job until its `state` is `succeeded` or `failed`. Its `log` has the end of the git and pip output. An adapter has one job at a time; another gets `409`. -An adapter's settings are at `/v1/platforms/adapters/{id}/config`, for administrators only. `GET` lists each setting, with secrets only marked `is_set`, and the inputs its setup takes. `PATCH` changes the settings sent and keeps the rest; `POST .../setup` runs the adapter's setup and replaces them all, with a file input sent as the file's text. `configured` in the adapter list says whether every required setting has a value. +An adapter's settings are at `/v1/platforms/adapters/{id}/config`, for administrators only. `GET` lists each setting, with secrets only marked `is_set`, and the inputs its setup takes. `PATCH` changes the settings sent and keeps the rest, and `POST .../setup` runs the adapter's setup and replaces them all, with a file input sent as the file's text. `configured` in the adapter list says whether every required setting has a value. ## Managing gateway clients -`/v1/gateway-clients/registry` lists every gateway client, with who last changed it, and takes the same `ETag` and `If-Match` rules as credentials (scopes `gc:read` and `gc:write`). `GET /v1/gateway-clients/registry/suggest?msisdn=` returns the details and PLMN candidates a create would use, to prefill a form for the administrator to confirm; see [how suggest works](../gateway-clients/manage.md#how-suggest-works). A disabled client disappears from the public `/v1/gateway-clients`. +`/v1/gateway-clients/registry` lists every gateway client, with who last changed it, and takes the same `ETag` and `If-Match` rules as credentials (scopes `gc:read` and `gc:write`). `GET /v1/gateway-clients/registry/suggest?msisdn=` returns the details and PLMN candidates a create would use, to prefill a form for the administrator to confirm. See [how suggest works](../gateway-clients/manage.md#how-suggest-works). A disabled client disappears from the public `/v1/gateway-clients`. ## Pagination @@ -46,13 +46,13 @@ List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. ## Publishing -`POST /v1/publications` takes JSON and tags payloads with protocol `https`. It needs no credentials: a payload publishes only if it decrypts with the keys of a linked account or, for an offline payload, passes the [offline publishing](../configuration/offline-publishing.md) rules. +`POST /v1/publications` takes JSON and tags payloads with protocol `https`. It needs no credentials ([what guards it](../explanation/security-model.md#what-travels-encrypted)). It only checks the payload's structure before queueing it. Decryption, payload type and adapter errors happen later, so they're logged on the server, not returned. ## Errors -Error bodies are `{"error": ""}`. The message is meant to be shown to users; the server logs more detail. +Error bodies are `{"error": ""}`. The message is meant to be shown to users. The server logs more detail. | Status | Meaning | | :--- | :--- | @@ -60,8 +60,8 @@ Error bodies are `{"error": ""}`. The message is meant to be shown to u | `401 Unauthorized` | Missing or invalid credentials | | `403 Forbidden` | Origin not allowed, missing scope, or a credential change beyond your scopes | | `404 Not Found` | The resource doesn't exist | -| `409 Conflict` | Already exists, in use, or changed by another request at the same moment; retry | -| `412 Precondition Failed` | `If-Match` doesn't match the current `ETag`; reload and retry | +| `409 Conflict` | Already exists, in use, or changed by another request at the same moment. Retry. | +| `412 Precondition Failed` | `If-Match` doesn't match the current `ETag`. Reload and retry. | | `422 Unprocessable Entity` | Validation error | | `428 Precondition Required` | `If-Match` header missing | | `429 Too Many Requests` | Rate limited by nginx ([limits](../operating/credentials.md#logging-in)) | diff --git a/install.sh b/install.sh index 22c8a81..4764326 100755 --- a/install.sh +++ b/install.sh @@ -5,8 +5,7 @@ set -Eeuo pipefail REPO_URL="${REPO_URL:-https://github.com/smswithoutborders/RelaySMS-Publisher.git}" -# Piped through curl there is no checkout to source lib.sh from, so clone one -# and run its install.sh with the same arguments. +# Piped through curl: clone the repo and run its install.sh with the same arguments. if [ ! -f "${BASH_SOURCE[0]:-}" ]; then [ "$EUID" -eq 0 ] || { echo "Run with sudo" >&2 @@ -358,9 +357,6 @@ configure_install_dir() { error "--instance-name '$INSTANCE_NAME' must start with a letter/digit and contain only letters, digits, underscores, and hyphens (max 32 chars)" fi - # Blocks top-level dirs (so a later `rm -rf "$INSTALL_DIR"` can't wipe a - # system directory) and sed-special characters (which would corrupt the - # unit-file templating in render_units). [[ "$INSTALL_DIR" =~ ^(/[A-Za-z0-9_.-]+){2,}/?$ ]] || error "$INSTALL_DIR is not a safe install location; use an absolute path with at least two segments, letters/digits/._- only (e.g. /opt/relaysms/relaysms-publisher)" @@ -378,7 +374,7 @@ configure_install_dir() { error "$INSTALL_DIR is an existing install owned by service user '$existing_owner', but this run resolved to '$SERVICE_USER'. Re-run install.sh with sudo as '$existing_owner' (SERVICE_USER follows the invoking sudo user), or use --install-dir to target a different directory." fi fi - # manage.sh has no --instance-name flag, so persist it here for reuse. + # Reuses the instance name saved by an earlier install. local existing_instance existing_instance="$(read_instance_name)" if [ -n "$existing_instance" ]; then @@ -402,7 +398,7 @@ configure_install_dir() { } # Same as prompt(), but a numbered menu instead of free text. Options are -# "label:value" pairs; default_idx is 1-based. +# "label:value" pairs, and default_idx is 1-based. # Usage: prompt_menu RESULTVAR "Question" default_idx "Label one:value1" "Label two:value2" prompt_menu() { local __resultvar="$1" question="$2" default_idx="$3" @@ -478,7 +474,7 @@ setup_service_user() { clone_repository() { log "Cloning repository" - # -c scopes the SSH-to-HTTPS rewrite to this command, not global git config. + # Fetches the SSH submodule URLs over HTTPS. if [ -d "$INSTALL_DIR/.git" ]; then log "Repository exists, updating" cd "$INSTALL_DIR" @@ -493,8 +489,6 @@ clone_repository() { git -c "url.https://github.com/.insteadOf=git@github.com:" \ clone --recurse-submodules -b "$BRANCH" "$REPO_URL" "$INSTALL_DIR" fi - # Git refuses to clone into a pre-existing non-empty dir, so this can't - # move earlier. $INSTALL_DIR is only guaranteed to exist past this point. echo "$INSTANCE_NAME" >"$INSTALL_DIR/.instance-name" } @@ -537,17 +531,6 @@ create_app_directories() { ensure_app_directories "$SERVICE_USER" } -run_config_check() { - log "Checking configuration" - # config reads .env itself the same way systemd does, so it is not sourced here. - (cd "$INSTALL_DIR" && sudo -u "$SERVICE_USER" venv/bin/python -m publisher config check) -} - -run_migrations() { - log "Running database migrations" - (cd "$INSTALL_DIR" && sudo -u "$SERVICE_USER" venv/bin/python -m alembic upgrade head) -} - install_services() { log "Installing systemd services" cd "$INSTALL_DIR" @@ -600,7 +583,7 @@ configure_docs_site() { local site="$DOCS_SITE_NAME" interactive=0 if [ -z "$site" ]; then - # Without --docs-site-name, ask only when the API has a site, since the manual's site needs nginx and certbot. + # Without --docs-site-name, ask only when the API has a site. [ -n "$SITE_NAME" ] || return 0 interactive=1 prompt site "Domain name to serve the manual on (e.g. docs.publisher.example.com, blank to skip): " "" @@ -642,7 +625,7 @@ install_site() { fi case "$do_cert" in n | N | no | NO) - log "Skipping certificate issuance; run manually: certbot --nginx -d $site" + log "Skipping certificate issuance. Run it manually: certbot --nginx -d $site" return ;; esac @@ -851,7 +834,6 @@ main() { configure_database configure_broker create_app_directories - # Only warn, since the operator may still be filling in .env. run_config_check || warn "Fix .env, then run: sudo $INSTALL_DIR/manage.sh check" run_migrations install_services @@ -861,8 +843,8 @@ main() { log "Installation complete" log " Config : $INSTALL_DIR/.env" - log " Manage : $INSTALL_DIR/manage.sh {start|stop|restart|status|logs|check|update|nginx}" - log " CLI : $INSTALL_DIR/publisher.sh {creds|platforms|gateway-clients|config} ... (or env|shell)" + log " Manage : $INSTALL_DIR/manage.sh COMMAND (run it without one to list them)" + log " CLI : $INSTALL_DIR/publisher.sh --help" } main "$@" diff --git a/manage.sh b/manage.sh index f4e0a95..39141a6 100755 --- a/manage.sh +++ b/manage.sh @@ -20,29 +20,15 @@ require_installed() { error "No installed service found. Run install.sh first, or for a local check: venv/bin/python -m publisher config check" } -run_migrations() { - require_installed - log "Running database migrations" - (cd "$INSTALL_DIR" && sudo -u "$SERVICE_USER" venv/bin/python -m alembic upgrade head) - # Registers adapters installed before the registry moved to the database, and - # moves their files out of their code directories. - (cd "$INSTALL_DIR" && sudo -u "$SERVICE_USER" venv/bin/python -m publisher platforms import) -} - -run_config_check() { - require_installed - log "Checking configuration" - # config reads .env itself the same way systemd does, so it is not sourced here. - (cd "$INSTALL_DIR" && sudo -u "$SERVICE_USER" venv/bin/python -m publisher config check) -} - cmd_check() { require_root + require_installed run_config_check } cmd_migrate() { require_root + require_installed run_migrations } @@ -96,7 +82,7 @@ logs_usage() { cat <<'EOF' Usage: manage.sh logs [OPTIONS] - -u, --unit NAME Service to show (rest|grpc|worker|beat|smtp), repeatable (default: all) + -u, --unit NAME Service to show (rest|grpc|worker|beat|smtp), repeatable (default: all) -n, --lines N Number of lines to show before following/exiting -s, --since DATE Only show entries at or after DATE (journalctl --since syntax) --no-follow Print the selected range and exit instead of tailing @@ -209,7 +195,7 @@ cmd_update() { cd "$INSTALL_DIR" local svc if [ "$pulled" = "0" ]; then - # Fetch first so a network failure leaves the services running. + # A failed fetch leaves the services running. git fetch for svc in "${SERVICE_UNITS[@]}"; do systemctl stop "$svc" @@ -217,7 +203,7 @@ cmd_update() { # Follows a force-pushed branch, and fails rather than overwrite local edits. git reset --keep "@{upstream}" git submodule update --init --recursive - # Finish with the manage.sh just checked out, so changed update steps apply now. + # Runs the rest of the update with the manage.sh just checked out. local args=(--pulled) [ "$migrate" = "1" ] && args+=(--migrate) exec "$INSTALL_DIR/manage.sh" update "${args[@]}" @@ -235,13 +221,11 @@ cmd_update() { export PATH="$CARGO_BIN:$PATH" make build - # Directories and units follow .env, which the migration may have just changed. - "$INSTALL_DIR/scripts/migrate-runtime-data.sh" require_installed ensure_app_directories "$SERVICE_USER" render_units "$SERVICE_USER" - # Each service fails only on the settings it uses, so restart all and report after. + # A config error is reported after every service restarts. local config_ok=1 run_config_check || config_ok=0 @@ -252,7 +236,7 @@ cmd_update() { systemctl restart "$svc" done systemctl start "$TARGET_UNIT" - # book/ exists only when install.sh set up a site for the manual, so other installs skip the rebuild. + # book/ exists only when install.sh set up a site for the manual. if [ -d book ]; then ./scripts/build-book.sh fi @@ -394,8 +378,6 @@ cmd_uninstall() { done systemctl daemon-reload - # Belt-and-suspenders against a top-level directory, even though - # INSTALL_DIR is always self-derived from this script's own location. [[ "$INSTALL_DIR" =~ ^(/[^/]+){2,}/?$ ]] || error "Refusing to remove '$INSTALL_DIR': not a safe path" rm -rf "$INSTALL_DIR" log "Uninstall complete" diff --git a/migrations/env.py b/migrations/env.py index b26aeb3..1ba2c46 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-3.0-only import sys from pathlib import Path diff --git a/migrations/versions/003_add_token_hashes_table.py b/migrations/versions/003_add_token_hashes_table.py index b2c5a5d..dc0bdef 100644 --- a/migrations/versions/003_add_token_hashes_table.py +++ b/migrations/versions/003_add_token_hashes_table.py @@ -32,8 +32,8 @@ def upgrade(): sa.Column("updated_at", sa.DateTime(), nullable=False), sa.Column("last_used_at", sa.DateTime(), nullable=True), ) - # MySQL can't index a BLOB/TEXT column without an explicit key length; - # mysql_length is ignored on other dialects. + # mysql_length sets the key length MySQL needs to index a BLOB column. Other + # dialects ignore it. op.create_index( "uq_token_hashes_token_hash", "token_hashes", diff --git a/migrations/versions/009_publication_stats_table.py b/migrations/versions/009_publication_stats_table.py index 1cf27d8..c278302 100644 --- a/migrations/versions/009_publication_stats_table.py +++ b/migrations/versions/009_publication_stats_table.py @@ -31,7 +31,7 @@ def upgrade() -> None: batch_op.alter_column( "platform_name", existing_type=sa.String(length=100), nullable=True ) - # Without existing_nullable, MySQL's MODIFY makes the column nullable. + # existing_nullable keeps the column NOT NULL through MySQL's MODIFY. batch_op.alter_column( "status", existing_type=sa.String(length=50), @@ -70,8 +70,7 @@ def downgrade() -> None: ) batch_op.add_column(sa.Column("source", sa.String(length=255), nullable=True)) - # Re-adding these columns as NOT NULL directly would fail on a non-empty - # table (no default to satisfy existing rows), so backfill first. + # Backfills the re-added columns before making them NOT NULL. publication_stats = sa.table( "publication_stats", sa.column("platform_name", sa.String), diff --git a/migrations/versions/012_add_admin_users_and_sessions.py b/migrations/versions/012_add_admin_users_and_sessions.py index 48defbe..fe18a22 100644 --- a/migrations/versions/012_add_admin_users_and_sessions.py +++ b/migrations/versions/012_add_admin_users_and_sessions.py @@ -55,6 +55,5 @@ def upgrade() -> None: def downgrade() -> None: - # Dropping the FK-backing index before its table fails on MySQL. op.drop_table("admin_sessions") op.drop_table("admin_users") diff --git a/migrations/versions/014_add_audit_events_table.py b/migrations/versions/014_add_audit_events_table.py index ef1f37b..70ece7f 100644 --- a/migrations/versions/014_add_audit_events_table.py +++ b/migrations/versions/014_add_audit_events_table.py @@ -16,7 +16,8 @@ branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None -# Every scope before audit:read; holders were administrators and should stay so. +# Every scope that existed before audit:read. Credentials holding all of them +# are administrators and get audit:read. _PREVIOUS_SCOPES = ( "stats:publications:read", "stats:publications:reasons", diff --git a/migrations/versions/019_add_gateway_clients_tables.py b/migrations/versions/019_add_gateway_clients_tables.py index 5a16ac6..d7ce5f6 100644 --- a/migrations/versions/019_add_gateway_clients_tables.py +++ b/migrations/versions/019_add_gateway_clients_tables.py @@ -1,17 +1,11 @@ -"""Add gateway_clients, importing the JSON registry +"""Add gateway_clients Revision ID: 019 Revises: 018 Create Date: 2026-10-07 00:00:00.000000 """ -import datetime -import json -import logging -import os -import uuid from collections.abc import Sequence -from pathlib import Path import sqlalchemy as sa from alembic import op @@ -22,58 +16,9 @@ branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None -logger = logging.getLogger(__name__) - -ROOT = Path(__file__).resolve().parents[2] -_FIELDS = ("msisdn", "country", "operator", "operator_code", "protocols") - - -def _registry_file() -> Path: - # Only .env files from before this migration set it. - return ROOT / os.environ.get( - "GATEWAY_CLIENTS_REGISTRY_FILE", "data/gateway_clients/registry.json" - ) - - -def _read(path: Path) -> list[dict]: - if not path.is_file(): - return [] - data = json.loads(path.read_text(encoding="utf-8") or "[]") - return list(data.values()) if isinstance(data, dict) else data - - -def _complete(record: dict, path: Path) -> bool: - if all(record.get(field) for field in _FIELDS): - return True - logger.warning("Skipping an incomplete entry in %s: %s", path, record) - return False - - -def _import(clients: sa.Table) -> None: - now = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - registry = _registry_file() - op.bulk_insert( - clients, - [ - { - "id": uuid.uuid4(), - **{field: record[field] for field in _FIELDS}, - "is_enabled": True, - "version": 1, - "created_at": now, - "updated_at": now, - } - for record in _read(registry) - if _complete(record, registry) - ], - ) - overrides = registry.parent / "mcc_mnc_overrides.json" - for record in _read(overrides): - logger.warning("MCC/MNC overrides are no longer used; not imported: %s", record) - def upgrade() -> None: - clients = op.create_table( + op.create_table( "gateway_clients", sa.Column("id", sa.Uuid(), primary_key=True), sa.Column("msisdn", sa.String(length=20), nullable=False), @@ -91,7 +36,6 @@ def upgrade() -> None: op.create_index( "uq_gateway_clients_msisdn", "gateway_clients", ["msisdn"], unique=True ) - _import(clients) def downgrade() -> None: diff --git a/migrations/versions/020_drop_ephemeral_keys_used_columns.py b/migrations/versions/020_drop_ephemeral_keys_used_columns.py new file mode 100644 index 0000000..9b572db --- /dev/null +++ b/migrations/versions/020_drop_ephemeral_keys_used_columns.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-3.0-only +"""Drop used and used_at from the ephemeral key tables + +Revision ID: 020 +Revises: 019 +Create Date: 2026-10-29 00:00:00.000000 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "020" +down_revision: str | None = "019" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +TABLES = ("server_ephemeral_keys", "client_ephemeral_keys") + + +def upgrade() -> None: + for table in TABLES: + with op.batch_alter_table(table) as batch: + batch.drop_index(f"ix_{table}_token_hash_id_used") + batch.drop_column("used_at") + batch.drop_column("used") + + +def downgrade() -> None: + for table in TABLES: + with op.batch_alter_table(table) as batch: + batch.add_column( + sa.Column( + "used", sa.Boolean(), nullable=False, server_default=sa.false() + ) + ) + batch.add_column(sa.Column("used_at", sa.DateTime(), nullable=True)) + batch.create_index( + f"ix_{table}_token_hash_id_used", ["token_hash_id", "used"] + ) diff --git a/observability/signoz/casting.yaml.template b/observability/signoz/casting.yaml.template index 4bbd2f0..219989d 100644 --- a/observability/signoz/casting.yaml.template +++ b/observability/signoz/casting.yaml.template @@ -5,9 +5,6 @@ # observability/signoz/casting.yaml (gitignored, contains real # credentials). See docs/operating/observability.md for the manual equivalent. # -# Left unset, Foundry defaults the Postgres fields to the literal string -# "signoz", so don't cast this template file directly. -# # Writes pours/deployment/ relative to the current directory (generated, # gitignored, do not hand-edit). apiVersion: v1alpha1 @@ -38,13 +35,11 @@ spec: spec: config: data: - # Foundry points this at the metastore Postgres container instead - # of the actual SigNoz backend, so the collector can never - # register as an opamp agent. + # Points the collector's opamp client at the SigNoz backend. opamp.yaml: | server_endpoint: ws://relaysms-publisher-signoz-signoz-0:4320/v1/opamp patches: - # Foundry publishes these on 0.0.0.0 by default; restrict to localhost. + # Publishes these ports on 127.0.0.1 only. - target: "deployment/compose.yaml" operations: - op: replace diff --git a/observability/signoz/docker-compose.override.yml b/observability/signoz/docker-compose.override.yml index e145fe7..3f47f2c 100644 --- a/observability/signoz/docker-compose.override.yml +++ b/observability/signoz/docker-compose.override.yml @@ -1,11 +1,9 @@ # Resource caps and hardening for Foundry's generated -# pours/deployment/compose.yaml. Not hand-editing the generated file, -# foundryctl cast overwrites it. +# pours/deployment/compose.yaml, which foundryctl cast overwrites. # -# Service names are derived from casting.yaml's metadata.name -# (relaysms-publisher-signoz), but Foundry could rename its own scheme in -# a future release. If `up -d` below errors about an unknown service, -# re-check with: +# Service names come from casting.yaml's metadata.name +# (relaysms-publisher-signoz). If `docker compose up -d` errors about an +# unknown service, list the generated names with: # docker compose -f pours/deployment/compose.yaml config --services services: ingester: diff --git a/protos/v3/publisher.proto b/protos/v3/publisher.proto index aad9d3c..e0a99d0 100644 --- a/protos/v3/publisher.proto +++ b/protos/v3/publisher.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package publisher.v3; -// Represents a public key message PublicKey { // The key ID uint32 key_id = 1; @@ -23,9 +22,9 @@ message GetOAuth2AuthorizationUrlRequest { string platform = 1; // Optional state parameter to prevent CSRF attacks string state = 2; - // Optional PKCE code verifier. Ignored if autogenerate_code_verifier is true. + // Optional PKCE code verifier string code_verifier = 3; - // If true, a PKCE code verifier is auto-generated and code_verifier is ignored + // If true and code_verifier is empty, a PKCE code verifier is auto-generated bool autogenerate_code_verifier = 4; // Optional redirect URL for the OAuth2 application string redirect_url = 5; @@ -247,7 +246,6 @@ message SyncKeysResponse { repeated PublicKey server_ephemeral_public_keys = 3; } -// Publisher service service Publisher { // RPC for getting the OAuth2 authorization URL rpc GetOAuth2AuthorizationUrl(GetOAuth2AuthorizationUrlRequest) returns (GetOAuth2AuthorizationUrlResponse); diff --git a/publisher.sh b/publisher.sh index 8378566..efa4737 100755 --- a/publisher.sh +++ b/publisher.sh @@ -55,7 +55,7 @@ cmd_env() { local var for var in AUTH_WEB_ORIGINS AUTH_SESSION_COOKIE_SECURE AUTH_SESSION_IDLE_MINUTES \ AUTH_SESSION_MAX_HOURS PLATFORMS_GITHUB_ORGS PLATFORMS_ADAPTERS_DIR \ - PLATFORMS_ADAPTERS_VENV_DIR PLATFORMS_ADAPTERS_ASSETS_DIR; do + PLATFORMS_ADAPTERS_VENV_DIR PLATFORMS_ADAPTERS_CONFIG_DIR PLATFORMS_ADAPTERS_STATE_DIR; do echo "$var = $(read_env_var "$var" "$ENV_FILE")" done } diff --git a/publisher/api/grpc/interceptors.py b/publisher/api/grpc/interceptors.py index 99016ef..c9e7f37 100644 --- a/publisher/api/grpc/interceptors.py +++ b/publisher/api/grpc/interceptors.py @@ -162,7 +162,7 @@ def _verify(self, context, method_name: str) -> bytes: except rrs.V1CryptographicError.FailedToDecrypt: raise _rejected("decryption failed") from None - # The signed method name stops a request being replayed against another RPC. + # Rejects a request replayed against another RPC. if request.method_name.decode() != method_name: raise _rejected("method name in payload does not match gRPC method") diff --git a/publisher/api/grpc/server.py b/publisher/api/grpc/server.py index cdecb53..ba3e49f 100644 --- a/publisher/api/grpc/server.py +++ b/publisher/api/grpc/server.py @@ -105,8 +105,6 @@ def serve() -> None: grpc_server = _build_server(grpc_config.max_workers) _bind_port(grpc_server) - # The handler only sets this flag: stopping inside it would hang if a second - # signal arrived while the server was stopping. stop_requested = threading.Event() def request_stop(signum: int, _frame) -> None: diff --git a/publisher/api/grpc/utils.py b/publisher/api/grpc/utils.py index 1b27d21..871adfb 100644 --- a/publisher/api/grpc/utils.py +++ b/publisher/api/grpc/utils.py @@ -23,8 +23,7 @@ def require_fields(request, *fields: str) -> None: - # Only for string, bytes and repeated fields: proto3 sends 0 and false as unset, - # so a valid 0 id would be rejected. + # Only for string, bytes and repeated fields: proto3 sends 0 and false as unset. for field in fields: if not getattr(request, field, None): raise InvalidArgument(f"Missing required field: {field}") @@ -63,7 +62,7 @@ def call_adapter(adapter: PlatformAdapter, method: str, request: Any) -> dict: """Call an adapter, turning its errors into gRPC errors. Invalid input, failed authentication and rate limits reach the client with the - adapter's message; any other error is logged and the client gets a generic one. + adapter's message. Any other error is logged and the client gets a generic one. """ try: return ipc.call(adapter, method, request) diff --git a/publisher/api/grpc/v3/exchange_pnba_code.py b/publisher/api/grpc/v3/exchange_pnba_code.py index 684ac86..6352818 100644 --- a/publisher/api/grpc/v3/exchange_pnba_code.py +++ b/publisher/api/grpc/v3/exchange_pnba_code.py @@ -57,7 +57,7 @@ def exchange_pnba_code_and_store( two_step_verification_enabled=True, platform=adapter.name, cat_id=adapter.cat_id, - message="two-steps verification is enabled and a password is required", + message="Two-step verification is enabled and a password is required", ) account_identifier = result["identifier"] diff --git a/publisher/api/rest/app.py b/publisher/api/rest/app.py index 3b6e833..270e32a 100644 --- a/publisher/api/rest/app.py +++ b/publisher/api/rest/app.py @@ -36,7 +36,7 @@ "connect-src 'self'" ) VALIDATION_ERROR = ( - "A parameter or body field is missing or invalid; the message names it." + "A parameter or body field is missing or invalid. The message names it." ) UNAUTHENTICATED = ( "No session cookie or Basic credentials, an expired session, or a wrong " @@ -88,7 +88,6 @@ def _validation_message(error: dict) -> str: value = str(error["input"]) if len(value) > MAX_ECHOED_INPUT_LENGTH: value = value[:MAX_ECHOED_INPUT_LENGTH] + "..." - # repr quotes and escapes control characters, so logs stay one line. message += f", got {value!r}" return message diff --git a/publisher/api/rest/v1/audit.py b/publisher/api/rest/v1/audit.py index 5856ca7..0258a3f 100644 --- a/publisher/api/rest/v1/audit.py +++ b/publisher/api/rest/v1/audit.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-3.0-only -"""The audit log of credential activity.""" +"""The audit log of logins and changes.""" import datetime diff --git a/publisher/api/rest/v1/auth.py b/publisher/api/rest/v1/auth.py index 4abab83..a077d38 100644 --- a/publisher/api/rest/v1/auth.py +++ b/publisher/api/rest/v1/auth.py @@ -29,7 +29,7 @@ SESSION_COOKIE_PATH = "/v1" SESSION_COOKIE_SAMESITE = "strict" BASIC_REALM = "relaysms" -# Basic auth runs on every request, so last_login_at writes are throttled. +# Basic auth records last_login_at at most this often. BASIC_LOGIN_RECORD_INTERVAL_SECONDS = 300 SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) PRIVATE_CACHE_CONTROL = "private, no-store" @@ -105,8 +105,7 @@ def _request_origin(request: Request) -> str | None: def _record_failed_login(db: Session, username: str, method: str) -> None: - # Recorded only for existing usernames: anyone can send unknown ones, which - # would fill the audit table. + # Recorded only for existing usernames. target = get_by_username(db, username) if target is None: return @@ -123,7 +122,7 @@ def _record_failed_login(db: Session, username: str, method: str) -> None: def check_origin(request: Request) -> None: origin = _request_origin(request) - # Browsers send Origin on cross-origin POSTs; none means a non-browser client. + # No Origin or Referer means a non-browser client. if origin is None: return own_origin = f"{request.url.scheme}://{request.url.netloc}" @@ -163,7 +162,7 @@ def authenticate_request( ) return AuthContext(credential=credential) - # No Basic challenge, because browsers would show their own login dialog. + # No WWW-Authenticate header: browsers would show their own login dialog. raise HTTPException(status_code=401, detail="Authentication required.") @@ -215,7 +214,7 @@ def login( response: Response, db: Session = Depends(get_db), ) -> CurrentCredential: - """Sets an HttpOnly session cookie for the web client; scripts use HTTP Basic.""" + """Sets an HttpOnly session cookie for the web client. Scripts use HTTP Basic.""" check_origin(request) credential = authenticate(db, body.username, body.password) diff --git a/publisher/api/rest/v1/creds.py b/publisher/api/rest/v1/creds.py index 0eaecaf..0db02fa 100644 --- a/publisher/api/rest/v1/creds.py +++ b/publisher/api/rest/v1/creds.py @@ -250,7 +250,7 @@ def reset_credential_password( context: AuthContext = Security(authorize, scopes=[Scope.CREDS_WRITE]), db: Session = Depends(get_db), ) -> CredentialWithPassword: - """New password shown once; sessions end. Needs If-Match. Scope: creds:write.""" + """New password shown once, and sessions end. Needs If-Match. Scope: creds:write.""" credential = _load(db, username) check_if_match(request, _etag(credential), "credential") with _model_errors( @@ -279,7 +279,7 @@ def revoke_credential_sessions( context: AuthContext = Security(authorize, scopes=[Scope.CREDS_WRITE]), db: Session = Depends(get_db), ) -> Response: - """Logs it out everywhere; Basic auth still works. Scope: creds:write.""" + """Logs it out everywhere. Basic auth still works. Scope: creds:write.""" credential = _load(db, username) with _model_errors( db, context.credential, AuditAction.CREDS_REVOKE_SESSIONS, credential diff --git a/publisher/api/rest/v1/params.py b/publisher/api/rest/v1/params.py index be7b307..992c226 100644 --- a/publisher/api/rest/v1/params.py +++ b/publisher/api/rest/v1/params.py @@ -44,8 +44,7 @@ def check_if_match(request: Request, current: str, what: str) -> None: if_match = request.headers.get("If-Match") if if_match is None: raise ApiError(428, "If-Match header required.") - # nginx gzip marks the ETag weak (W/"..."). Ours come from the row version, not - # the body, so a weak one still matches. + # A weak ETag (W/"...", e.g. from nginx gzip) matches too. if if_match.strip().removeprefix("W/") != current: raise ApiError( 412, diff --git a/publisher/api/rest/v1/platform_adapters.py b/publisher/api/rest/v1/platform_adapters.py index 8892bdd..cc6da4a 100644 --- a/publisher/api/rest/v1/platform_adapters.py +++ b/publisher/api/rest/v1/platform_adapters.py @@ -105,7 +105,7 @@ def _adapter_errors(context: AuthContext): except (AdapterInUseError, AdapterConflictError) as e: raise ApiError(409, str(e), log=f"{by}: {e}") from None except AdapterFaultError as e: - message = "The adapter or its files are broken; see the server log." + message = "The adapter or its files are broken. See the server log." raise ApiError(502, message, log=f"{by}: {e}") from None except AdapterError as e: raise ApiError(400, str(e), log=f"{by}: {e}") from None @@ -184,7 +184,7 @@ def delete_adapter( ) -> Response: """Needs If-Match. Scope: platforms:write. - Refused while accounts are linked through it; disable it instead. + Refused while accounts are linked through it. Disable it instead. """ adapter = _load(db, adapter_id) check_if_match(request, _etag(adapter), "adapter") @@ -320,8 +320,7 @@ def _queue( try: run_adapter_job.delay(str(job.id)) except Exception as e: - # Fail the job now. Otherwise it stays active and blocks every other job for - # the adapter until cleanup fails it as abandoned. + # Fail the job now, freeing the adapter for other jobs. jobs.finish(db, job.id, state="failed", log=[f"Queueing failed: {e}"]) db.commit() raise ApiError( @@ -353,7 +352,7 @@ def install_adapter( context: AuthContext = Security(authorize, scopes=[Scope.PLATFORMS_WRITE]), db: Session = Depends(get_db), ) -> AdapterJobInfo: - """Queues an install; poll the job at Location. Administrators only.""" + """Queues an install. Poll the job at Location. Administrators only.""" _require_administrator(context, "Installing and updating adapters") _require_allowed(body.source_url) adapter_id = manager.adapter_id(body.source_url) @@ -395,7 +394,7 @@ def update_adapter_version( context: AuthContext = Security(authorize, scopes=[Scope.PLATFORMS_WRITE]), db: Session = Depends(get_db), ) -> AdapterJobInfo: - """Queues a move to a version tag; poll the job at Location. Administrators only.""" + """Queues a move to a version tag. Poll the job at Location. Administrators only.""" _require_administrator(context, "Installing and updating adapters") adapter = _load(db, adapter_id) _require_allowed(adapter.source_url) diff --git a/publisher/api/rest/v1/publications.py b/publisher/api/rest/v1/publications.py index 52f74da..a32fce2 100644 --- a/publisher/api/rest/v1/publications.py +++ b/publisher/api/rest/v1/publications.py @@ -26,9 +26,9 @@ def create_publications(body: PublishRestContentRequest) -> PublishContentRespon """Queues an SMS payload for publishing. Multi-part payloads are kept until every segment arrives, then published. Payloads - are tagged with protocol `https`; see Offline publishing in the manual for - `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors - happen later in the queue, so they're logged, not returned. + are tagged with protocol `https`. See Offline publishing in the manual for + `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors are + logged, not returned. """ try: publications.validate(body.text) diff --git a/publisher/api/rest/v1/schemas.py b/publisher/api/rest/v1/schemas.py index e11dc8a..4678772 100644 --- a/publisher/api/rest/v1/schemas.py +++ b/publisher/api/rest/v1/schemas.py @@ -12,7 +12,7 @@ from relaysms_adapter_sdk.manifest import FieldType USERNAME_FIELD = Field( - None, description="Username; null for the CLI or a deleted credential." + None, description="Username, or null for the CLI or a deleted credential." ) @@ -42,8 +42,7 @@ class OAuthClientMetadata(BaseModel): client_name: str client_uri: str scope: str - # The same for every atproto web client, so the adapter's settings leave them - # out and these defaults fill them in. + # Defaults for every atproto web client, used when the adapter's settings omit them. dpop_bound_access_tokens: bool = True application_type: str = "web" grant_types: list[str] = ["authorization_code", "refresh_token"] @@ -67,7 +66,7 @@ class PublishContentResponse(BaseModel): message: str | None = None error: str | None = Field( default=None, - description="Always null here; errors come back with a 4xx status.", + description="Always null here. Errors come back with an error status.", ) @@ -75,22 +74,26 @@ class PublicationStat(BaseModel): id: int platform_name: str | None = None protocol: str | None = None - status: str + status: str = Field( + ..., + description="published, failed or rejected (refused by the offline " + "publishing settings).", + ) country_code: str | None = None failure_reason: str | None = Field( None, - description="Why it failed, as a short code such as payload_malformed or " - "adapter_rate_limited.", + description="Why it failed or was rejected, as a short code such as " + "payload_malformed or offline_tag_invalid.", ) created_at: datetime.datetime class PageLinks(BaseModel): next: str | None = Field( - None, description="URL of the next (older) page; null on the last page." + None, description="URL of the next (older) page, or null on the last page." ) prev: str | None = Field( - None, description="URL of the previous (newer) page; null on the first page." + None, description="URL of the previous (newer) page, or null on the first page." ) @@ -176,7 +179,7 @@ class AuditEventInfo(BaseModel): actor: str | None = Field( None, validation_alias="actor_username", - description="Username at the time; null for the CLI or a failed login.", + description="Username at the time, or null for the CLI or a failed login.", ) action: str = Field(..., description="e.g. creds.update") target: str | None = Field( @@ -196,7 +199,9 @@ class PlatformAdapterInfo(PlatformManifest): id: str source_url: str tag: str | None = Field( - None, description="Null for an adapter registered by `platforms import`." + None, + description="Null for an adapter registered before tags were recorded, " + "until its first update.", ) commit: str enabled: bool @@ -316,14 +321,14 @@ class AdapterSetting(AdapterField): class AdapterConfig(BaseModel): settings: list[AdapterSetting] setup: list[AdapterField] = Field( - ..., description="Inputs POST .../setup takes; empty if it has no setup." + ..., description="Inputs POST .../setup takes, or empty if it has no setup." ) class AdapterConfigUpdate(BaseModel): model_config = ConfigDict(extra="forbid") settings: dict[str, Any] = Field( - ..., description="Settings to change; others are kept, and null removes one." + ..., description="Settings to change. Others are kept, and null removes one." ) diff --git a/publisher/api/rest/v1/stats.py b/publisher/api/rest/v1/stats.py index 06835b8..b7c5cba 100644 --- a/publisher/api/rest/v1/stats.py +++ b/publisher/api/rest/v1/stats.py @@ -120,11 +120,10 @@ def summarize_publication_stats( ), db: Session = Depends(get_db), ) -> PublicationStatsSummary: - """Counts per group; last 30 days by default. Scope: stats:publications:read. + """Counts per group (last 30 days by default). Scope: stats:publications:read. Grouping by `failure_reason` also needs stats:publications:reasons. Groups sort by - period, then count, highest first. Periods with no rows are left out; chart them - as 0. + period, then count, highest first. Periods with no rows are left out. """ columns = list(dict.fromkeys(item.value for item in group_by)) if "failure_reason" in columns: diff --git a/publisher/cli/creds.py b/publisher/cli/creds.py index 7308470..43f0b72 100644 --- a/publisher/cli/creds.py +++ b/publisher/cli/creds.py @@ -23,8 +23,8 @@ def _show_password(username, password): click.echo(f"Username : {username}") click.echo(f"Password : {password}") click.echo( - "Store this password now (e.g. in a password manager); " - "it is not stored and can't be shown again.", + "Store this password now (e.g. in a password manager). " + "It is not stored and can't be shown again.", err=True, ) @@ -135,7 +135,7 @@ def reset_password(username): credential = credentials.get_or_raise(db, username) password = credentials.reset_password(db, credential) - click.echo("Password reset; existing sessions were ended.") + click.echo("Password reset. Existing sessions were ended.") _show_password(credential.username, password) @@ -146,7 +146,7 @@ def disable(username): with session() as db: credentials.update(db, credentials.get_or_raise(db, username), active=False) - click.echo(f"Credential {username} disabled; existing sessions were ended.") + click.echo(f"Credential {username} disabled. Existing sessions were ended.") @cli.command() diff --git a/publisher/cli/db.py b/publisher/cli/db.py index f77c1fb..dced4e3 100644 --- a/publisher/cli/db.py +++ b/publisher/cli/db.py @@ -13,7 +13,7 @@ @contextmanager def session() -> Generator[Session]: - """Commit on success; show domain errors as CLI errors.""" + """Commit on success and show domain errors as CLI errors.""" try: with get_session() as db: yield db diff --git a/publisher/cli/platforms.py b/publisher/cli/platforms.py index c681c7a..08ee3aa 100644 --- a/publisher/cli/platforms.py +++ b/publisher/cli/platforms.py @@ -93,7 +93,7 @@ def enable(name, proto_id, cat_id): @click.argument("name") @_filter_options def disable(name, proto_id, cat_id): - """Hide an adapter from users; it can still revoke tokens.""" + """Hide an adapter from users. It can still revoke tokens.""" with session() as db: manager.set_enabled(db, _find_one(db, name, proto_id, cat_id), False) click.echo(f"Adapter {name!r} disabled.") @@ -144,7 +144,7 @@ def update(name, proto_id, cat_id, tag): ids = [adapter.id for adapter in adapters] failed = False - # One transaction each, so one bad adapter doesn't undo the others. + # A failed update doesn't undo the others. for adapter_id in ids: try: with session() as db: @@ -213,7 +213,7 @@ def config_group(): @click.argument("name") @_filter_options def config_show(name, proto_id, cat_id): - """Show an adapter's settings; secrets only show whether they're set.""" + """Show an adapter's settings. Secrets show only whether they're set.""" with session() as db: adapter = _find_one(db, name, proto_id, cat_id) settings = adapter_config.show(adapter) @@ -260,7 +260,7 @@ def config_set(name, values, unset, proto_id, cat_id): "inputs", multiple=True, metavar="KEY=VALUE", - help="Repeatable; KEY=@PATH for a file. Asks for required inputs not given.", + help="Repeatable. KEY=@PATH reads a file. Asks for required inputs not given.", ) @_filter_options def setup(name, inputs, proto_id, cat_id): @@ -309,14 +309,3 @@ def _shown_value(setting: adapter_config.Setting) -> str: return "-" value = setting.value return ", ".join(value) if isinstance(value, list) else str(value) - - -@cli.command(name="import") -def import_command(): - """Register adapter directories on disk and move their files out of the code.""" - with session() as db: - imported = [adapter.name for adapter in manager.import_from_disk(db)] - moved = [adapter.name for adapter in manager.move_files_out_of_code(db)] - click.echo(f"Imported {len(imported)} adapter(s). {' '.join(imported)}".rstrip()) - if moved: - click.echo(f"Moved the files of {len(moved)} adapter(s). {' '.join(moved)}") diff --git a/publisher/cli/seed.py b/publisher/cli/seed.py index 237af43..56edea9 100644 --- a/publisher/cli/seed.py +++ b/publisher/cli/seed.py @@ -32,13 +32,14 @@ "token_not_found", "key_unavailable", "payload_malformed", - "offline_tag_invalid", + "segments_missing", "adapter_token_invalid", "adapter_rate_limited", "adapter_upstream", "unexpected_error", ) -FAILURE_RATE = 0.2 +REJECTION_REASONS = ("protocol_not_allowed", "offline_tag_invalid") +UNPUBLISHED_RATE = 0.2 DISABLED_RATE = 0.1 BATCH_SIZE = 1000 # name, display name, protocol, category (0 email, 1 message, 2 text). @@ -53,16 +54,19 @@ def _random_stat(now: datetime.datetime, days: int) -> dict: - failed = random.random() < FAILURE_RATE + status, reason = "published", None + if random.random() < UNPUBLISHED_RATE: + reason = random.choice(FAILURE_REASONS + REJECTION_REASONS) + status = "rejected" if reason in REJECTION_REASONS else "failed" return dict( - status="failed" if failed else "published", + status=status, protocol=random.choice(PROTOCOLS), - # Some failures happen before the platform is known. + # Failures and rejections can happen before the platform is known. platform_name=( - None if failed and random.random() < 0.3 else random.choice(PLATFORMS) + None if reason and random.random() < 0.3 else random.choice(PLATFORMS) ), country_code=random.choice(COUNTRIES), - failure_reason=random.choice(FAILURE_REASONS) if failed else None, + failure_reason=reason, created_at=now - datetime.timedelta(seconds=random.uniform(0, days * 86400)), ) @@ -136,7 +140,7 @@ def platforms(): PlatformAdapter( id=adapter_id(url), source_url=url, - # No tag, so platforms update installs the newest release. + # With no tag, platforms update installs the newest release. commit=secrets.token_hex(20), name=name, display_name=display_name, @@ -173,8 +177,7 @@ def gateway_clients_command(count): while added < count: msisdn = _random_mobile(random.choice(COUNTRIES)) suggestion = gateway_clients.suggest(msisdn) - # Takes the first candidate when the operator is ambiguous, as an - # administrator would. + # Takes the first candidate when the operator is ambiguous. candidate = suggestion.candidates[0] try: client = gateway_clients.create( diff --git a/publisher/config.py b/publisher/config.py index d7bb735..ea20c52 100644 --- a/publisher/config.py +++ b/publisher/config.py @@ -52,10 +52,10 @@ def get_str(self, name: str, default: str) -> str: ... def get_str(self, name: str, default: None = None) -> str | None: ... def get_str(self, name: str, default: str | None = None) -> str | None: value = self.env.get(name, "").strip() - # Drop one pair of surrounding quotes, since docker run --env-file keeps them. + # Drop one pair of surrounding quotes, e.g. from docker run --env-file. if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": value = value[1:-1].strip() - # An empty value counts as unset, since template.env leaves options blank. + # An empty value counts as unset. return value or default def get_required(self, name: str, required: bool) -> str | None: @@ -133,7 +133,7 @@ def get_secret(self, name: str, required: bool = False) -> str | None: return self.get_str(name) if self.get_key(name, required) else None def get_path(self, name: str, default: Path) -> Path: - """Read a path; a relative one is relative to the install directory.""" + """Read a path, resolving a relative one against the install directory.""" path = Path(self.get_str(name) or default).expanduser() return path if path.is_absolute() else ROOT / path @@ -346,7 +346,6 @@ class OfflinePublishConfig(Section): @classmethod def load(cls, read: _Reader) -> Self: allowed = read.get_list("OFFLINE_PUBLISH_ALLOWED_PROTOCOLS") - # Without the secret, anyone can publish offline payloads over https. https_allowed = not allowed or "https" in allowed return cls( allowed_protocols=allowed, @@ -463,7 +462,7 @@ def load(cls, read: _Reader) -> Self: if "*" in web_origins: read.fail( "AUTH_WEB_ORIGINS", - "must list exact origins; '*' can't carry credentials", + "must list exact origins, not '*'", ) return cls( idle_timeout=datetime.timedelta( diff --git a/publisher/credentials.py b/publisher/credentials.py index 082484d..48e0154 100644 --- a/publisher/credentials.py +++ b/publisher/credentials.py @@ -123,7 +123,7 @@ def _claim(session: Session, credential: Credential) -> None: ) if result.rowcount != 1: raise CredentialConflictError( - f"Credential {credential.username!r} was changed concurrently; retry" + f"Credential {credential.username!r} was changed concurrently. Retry." ) set_committed_value(credential, "version", credential.version + 1) @@ -301,7 +301,6 @@ def delete( def authenticate(session: Session, username: str, password: str) -> Credential | None: credential = get_by_username(session, username) if credential is None: - # Hash anyway so response timing doesn't reveal which usernames exist. with contextlib.suppress(VerificationError): password_hasher.verify(_dummy_hash(), password) return None diff --git a/publisher/db/__init__.py b/publisher/db/__init__.py index 5960e63..6b968ce 100644 --- a/publisher/db/__init__.py +++ b/publisher/db/__init__.py @@ -37,7 +37,7 @@ def connect(): conn.execute("SELECT count(*) FROM sqlite_master;") except sqlcipher3.DatabaseError: # pyright: ignore[reportAttributeAccessIssue] conn.close() - raise ValueError("Invalid hex key or database file is corrupted.") from None + raise ValueError("Wrong key, or the database file is corrupted.") from None return conn @@ -126,11 +126,11 @@ def _ensure_postgres_database(server: ServerDatabaseConfig) -> None: def _create_engine() -> Engine: - # Read when the engine is built, since many processes import db but never use it. + # Read when the engine is first built, not at import. database = DatabaseConfig.get() if database.dialect == "sqlite" and database.sqlite_path == ":memory:": - # One shared connection, so every session sees the same in-memory database. + # Every session shares one connection to the same in-memory database. return create_engine( "sqlite:///:memory:", connect_args={"check_same_thread": False}, @@ -142,15 +142,15 @@ def _create_engine() -> Engine: if database.dialect == "mysql": if database.encryption_enabled: logger.debug( - "Database encryption enabled - ensure TDE is configured on " + "Database encryption enabled: ensure TDE is configured on the " "MySQL/MariaDB server" ) _ensure_mysql_database(database.mysql) elif database.dialect == "postgres": if database.encryption_enabled: logger.debug( - "Database encryption enabled - Postgres has no built-in TDE, " - "use disk-level encryption on the server" + "Database encryption enabled: Postgres has no built-in TDE. " + "Use disk-level encryption on the server" ) _ensure_postgres_database(database.postgres) @@ -171,8 +171,7 @@ def _create_engine() -> Engine: creator=_make_sqlcipher3_creator( database.sqlite_path, database.encryption_key ), - # Otherwise SQLAlchemy wraps only sqlite3's errors, and sqlcipher3's - # escape unwrapped, e.g. past `except IntegrityError`. + # Makes SQLAlchemy wrap sqlcipher3's errors, e.g. as IntegrityError. module=sqlcipher3_dbapi, echo=sql_echo, poolclass=QueuePool, diff --git a/publisher/db/pagination.py b/publisher/db/pagination.py index 5c8dfcd..32ff80b 100644 --- a/publisher/db/pagination.py +++ b/publisher/db/pagination.py @@ -71,7 +71,7 @@ def paginate[T]( """Run stmt for the page after cursor. Rows need the two columns as attributes.""" backward = cursor is not None and cursor.direction == "prev" if cursor is not None: - # OR form: row-value comparisons aren't consistent across databases. + # Compares (time, id) with OR and AND, not a row-value comparison. beyond = operator.gt if backward else operator.lt stmt = stmt.where( or_( diff --git a/publisher/db/types.py b/publisher/db/types.py index 50cb104..2591197 100644 --- a/publisher/db/types.py +++ b/publisher/db/types.py @@ -26,7 +26,7 @@ def as_utc(value: datetime.datetime) -> datetime.datetime: ... @overload def as_utc(value: None) -> None: ... def as_utc(value: datetime.datetime | None) -> datetime.datetime | None: - """Naive values are assumed to be UTC.""" + """Return value as aware UTC, treating a naive one as UTC.""" if value is None: return None if value.tzinfo is None: @@ -67,7 +67,7 @@ def __init__(self, unit: str, column): def _sql_string(value: str): - # Inlined, not bound: Postgres needs GROUP BY to match SELECT exactly. + # Renders the string inline so GROUP BY and SELECT match exactly on Postgres. return literal_column(f"'{value}'") @@ -111,7 +111,7 @@ def _date_bucket_sqlite(element, compiler, **kw): def _field_encryption_key() -> bytes | None: - # Read on each use, so importing models does not need the database settings. + # Read on each use, not at import. database = DatabaseConfig.get() return database.field_encryption_key if database.field_encryption_enabled else None diff --git a/publisher/errors.py b/publisher/errors.py index 225e125..8afb648 100644 --- a/publisher/errors.py +++ b/publisher/errors.py @@ -3,7 +3,7 @@ class PublisherError(Exception): - """Base for domain errors; platform_name is set when the platform is known.""" + """Base for domain errors, with platform_name set when the platform is known.""" def __init__(self, message: str, *, platform_name: str | None = None): super().__init__(message) diff --git a/publisher/gateway_clients/manager.py b/publisher/gateway_clients/manager.py index 2236fa2..c534217 100644 --- a/publisher/gateway_clients/manager.py +++ b/publisher/gateway_clients/manager.py @@ -62,7 +62,7 @@ def _candidates(rows: list[dict]) -> list[Candidate]: def suggest(msisdn: str) -> Suggestion: - """Best-effort details for an MSISDN; operator_code only when unambiguous.""" + """Best-effort details for an MSISDN, with operator_code only when unambiguous.""" msisdn = msisdn.strip() try: number = phonenumbers.parse(msisdn, None) @@ -94,7 +94,7 @@ def suggest(msisdn: str) -> Suggestion: def check_msisdn(msisdn: str) -> str: - """Return it stripped, or raise if it isn't in E.164 format.""" + """Return the MSISDN stripped, or raise GatewayClientError if it isn't E.164.""" msisdn = msisdn.strip() if not MSISDN_PATTERN.match(msisdn): raise GatewayClientError(f"MSISDN {msisdn!r} isn't in E.164 format") @@ -125,7 +125,7 @@ def _flush(session: Session, client: GatewayClient) -> None: ) from None except StaleDataError: raise GatewayClientConflictError( - f"Gateway client {msisdn!r} was changed concurrently; retry" + f"Gateway client {msisdn!r} was changed concurrently. Retry." ) from None @@ -166,7 +166,7 @@ def create( codes = ", ".join(sorted({c.operator_code for c in suggestion.candidates})) message = f"Multiple PLMNs match operator {operator!r}: {codes}." raise GatewayClientError( - f"{message} Supply the missing details; suggest lists the candidates." + f"{message} Supply the missing details (suggest lists the candidates)." ) client = GatewayClient( @@ -200,7 +200,7 @@ def update( protocols: list[str] | None = None, actor: Credential | None = None, ) -> None: - """Change the given fields; None leaves a field as it is.""" + """Change the given fields, leaving those passed as None.""" _check(operator_code=operator_code, protocols=protocols) new = { "country": country, diff --git a/publisher/gateway_clients/mcc_mnc.py b/publisher/gateway_clients/mcc_mnc.py index 1c05aa2..4fa76b0 100644 --- a/publisher/gateway_clients/mcc_mnc.py +++ b/publisher/gateway_clients/mcc_mnc.py @@ -30,8 +30,8 @@ ) # Snapshot rows that aren't operators. _NOT_OPERATORS = frozenset({"", "failed calls", "fix line", "unknown"}) -# A word in at least this many of a region's network names is common, so sharing -# it is only a weak match. +# A word in at least this many of a region's network names counts as common and +# gives only a weak match. _COMMON_IN_REGION = 3 diff --git a/publisher/keys.py b/publisher/keys.py index e3aa85b..a59cded 100644 --- a/publisher/keys.py +++ b/publisher/keys.py @@ -4,9 +4,10 @@ import hashlib import logging import secrets +from typing import cast from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey -from sqlalchemy import delete, func, insert, select +from sqlalchemy import CursorResult, delete, func, insert, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session @@ -43,7 +44,7 @@ class TokenVerificationError(KeyManagementError): def initialize_server_identity_keys(session: Session, count: int = 256) -> None: - """Create database identity keys if they don't exist yet.""" + """Create the server identity keys if none exist.""" existing = session.execute(select(func.count(ServerIdentityKey.id))).scalar_one() if existing > 0: logger.info("Server identity keys exist (%d found), skipping", existing) @@ -76,7 +77,7 @@ def create_token_pools_and_encrypt( ) -> tuple[bytes, int, list[bytes]]: """Create the token's hash and key pools, and encrypt the token to one slot. - Slots 0-15 are reserved, so the token's slot is picked from 16-255. + The token's slot is picked from 16-255 (0-15 are reserved). """ token_hash, raw_token = create_token_hash(session, token_pk_id) kid_index = secrets.randbelow(240) + 16 @@ -100,7 +101,7 @@ def create_token_pools_and_encrypt( def sync_token_pools( session: Session, token_hash: TokenHash, client_public_keys: list ) -> list[bytes]: - """Replace the token's key pools; return the server public keys by slot.""" + """Replace the token's key pools and return the server public keys by slot.""" session.execute( delete(ServerEphemeralKey).where( ServerEphemeralKey.token_hash_id == token_hash.id @@ -133,7 +134,6 @@ def _insert_pools( "key_index": i, "private_key": kp.private_bytes_raw(), "public_key": server_public_keys[i], - "used": False, } for i, kp in enumerate(server_keypairs) if i != skip @@ -146,7 +146,6 @@ def _insert_pools( "token_hash_id": token_hash_id, "key_index": k.key_id, "public_key": k.public_key, - "used": False, } for k in client_public_keys if k.key_id != skip @@ -155,41 +154,36 @@ def _insert_pools( return server_keypairs, server_public_keys -def pop_keys( - session: Session, token_hash_id: int, key_id: int -) -> tuple[bytes, bytes, bytes, bytes]: - """Delete and return a slot's keys: ss_kid, es_kid, es_kid_pk, ec_kid_pk.""" - se_row = session.execute( - delete(ServerEphemeralKey) - .where( - ServerEphemeralKey.token_hash_id == token_hash_id, - ServerEphemeralKey.key_index == key_id, +def _pop_slot[S: (ServerEphemeralKey, ClientEphemeralKey)]( + session: Session, model: type[S], token_hash_id: int, key_id: int, label: str +) -> S: + """Delete and return a slot's row, or raise KeyUnavailableError if it's gone.""" + row = session.scalars( + select(model).where( + model.token_hash_id == token_hash_id, model.key_index == key_id ) - .returning(ServerEphemeralKey.private_key, ServerEphemeralKey.public_key) ).first() - if not se_row: - logger.error( - "Server ephemeral key unavailable: token_hash_id=%s, kid=%s", - token_hash_id, - key_id, + if row is not None: + deleted = cast( + CursorResult, session.execute(delete(model).where(model.id == row.id)) ) - raise KeyUnavailableError("Server ephemeral key already used or missing") + if deleted.rowcount: + return row + logger.error( + "%s ephemeral key unavailable: token_hash_id=%s, kid=%s", + label, + token_hash_id, + key_id, + ) + raise KeyUnavailableError(f"{label} ephemeral key already used or missing") - ce_row = session.execute( - delete(ClientEphemeralKey) - .where( - ClientEphemeralKey.token_hash_id == token_hash_id, - ClientEphemeralKey.key_index == key_id, - ) - .returning(ClientEphemeralKey.public_key) - ).first() - if not ce_row: - logger.error( - "Client ephemeral key unavailable: token_hash_id=%s, kid=%s", - token_hash_id, - key_id, - ) - raise KeyUnavailableError("Client ephemeral key already used or missing") + +def pop_keys( + session: Session, token_hash_id: int, key_id: int +) -> tuple[bytes, bytes, bytes, bytes]: + """Delete and return a slot's keys: ss_kid, es_kid, es_kid_pk, ec_kid_pk.""" + se_row = _pop_slot(session, ServerEphemeralKey, token_hash_id, key_id, "Server") + ce_row = _pop_slot(session, ClientEphemeralKey, token_hash_id, key_id, "Client") ss_kid = get_private_key(session, key_id).private_bytes_raw() logger.debug( diff --git a/publisher/log.py b/publisher/log.py index 2543afa..a2440c0 100644 --- a/publisher/log.py +++ b/publisher/log.py @@ -10,7 +10,7 @@ def setup_logging() -> None: """Configure the root logger at LOG_LEVEL.""" - # journald adds its own timestamp to each line. + # Leave out the timestamp under journald, which adds its own. under_journald = bool(os.getenv("JOURNAL_STREAM")) and not sys.stderr.isatty() log_format = "%(name)s - %(levelname)s - %(message)s" if not under_journald: diff --git a/publisher/models/audit_event.py b/publisher/models/audit_event.py index 242aa11..437ce0d 100644 --- a/publisher/models/audit_event.py +++ b/publisher/models/audit_event.py @@ -77,11 +77,10 @@ class AuditEvent(Base): occurred_at: Mapped[datetime.datetime] = mapped_column(UTCDateTime, default=utc_now) # NULL when no credential acted: the CLI, or a failed login. actor_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, default=None) - # The usernames are copied so events still read right after a deletion. + # Kept when the actor's credential is deleted. actor_username: Mapped[str | None] = mapped_column( String(MAX_USERNAME_LENGTH), default=None ) - # Plain strings, so adding an action needs no migration and old ones still list. action: Mapped[str] = mapped_column(String(64)) target_id: Mapped[str | None] = mapped_column(String(64), default=None) target_label: Mapped[str | None] = mapped_column(String(255), default=None) @@ -126,7 +125,7 @@ def list_events( since: datetime.datetime | None = None, until: datetime.datetime | None = None, ) -> Page[AuditEvent]: - """Newest first, only from the areas whose scope is in scopes.""" + """List events newest first, from the areas whose scope is in scopes.""" areas = [area for area, scope in AREA_SCOPES.items() if scope in scopes] if not areas: return Page(items=[], next_cursor=None, prev_cursor=None) diff --git a/publisher/models/client_ephemeral_key.py b/publisher/models/client_ephemeral_key.py index 0e32892..38c3fc0 100644 --- a/publisher/models/client_ephemeral_key.py +++ b/publisher/models/client_ephemeral_key.py @@ -4,7 +4,7 @@ import datetime from typing import TYPE_CHECKING -from sqlalchemy import ForeignKey, Index, LargeBinary, UniqueConstraint +from sqlalchemy import ForeignKey, LargeBinary, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from publisher.db import Base @@ -23,12 +23,10 @@ class ClientEphemeralKey(Base): ) key_index: Mapped[int] = mapped_column() public_key: Mapped[bytes] = mapped_column(LargeBinary(32)) - used: Mapped[bool] = mapped_column(default=False) created_at: Mapped[datetime.datetime] = mapped_column(default=utc_now) updated_at: Mapped[datetime.datetime] = mapped_column( default=utc_now, onupdate=utc_now ) - used_at: Mapped[datetime.datetime | None] = mapped_column(default=None) token_hash: Mapped["TokenHash"] = relationship( "TokenHash", back_populates="client_keys" @@ -38,5 +36,4 @@ class ClientEphemeralKey(Base): UniqueConstraint( "token_hash_id", "key_index", name="uq_client_keys_token_hash_id_key_index" ), - Index("ix_client_ephemeral_keys_token_hash_id_used", "token_hash_id", "used"), ) diff --git a/publisher/models/credential_session.py b/publisher/models/credential_session.py index 9d6a2ee..bcbee00 100644 --- a/publisher/models/credential_session.py +++ b/publisher/models/credential_session.py @@ -106,8 +106,7 @@ def get_active(session: Session, raw_token: str) -> CredentialSession | None: return None credential = credential_session.credential - # None when the credential was deleted while this session was being created, - # since SQLite doesn't enforce the foreign key. + # None on SQLite if the credential was deleted while this session was being created. if credential is None or not credential.is_active: return None if credential_session.session_version != credential.session_version: diff --git a/publisher/models/gateway_client.py b/publisher/models/gateway_client.py index 5e02ea6..b152924 100644 --- a/publisher/models/gateway_client.py +++ b/publisher/models/gateway_client.py @@ -44,7 +44,10 @@ def find( operator: str | None = None, include_disabled: bool = False, ) -> list[GatewayClient]: - """Enabled only, unless include_disabled. Country and operator ignore case.""" + """Return matching clients, enabled only unless include_disabled. + + Country and operator ignore case. + """ stmt = select(GatewayClient).order_by( GatewayClient.country, GatewayClient.operator, GatewayClient.msisdn ) @@ -52,7 +55,7 @@ def find( stmt = stmt.where(GatewayClient.msisdn == msisdn.strip()) if not include_disabled: stmt = stmt.where(GatewayClient.is_enabled) - # Filtered here: SQLite's lower() only folds ASCII. + # Matched in Python, which also folds non-ASCII case. country, operator = ( term.strip().casefold() if term else None for term in (country, operator) ) diff --git a/publisher/models/payload_segment.py b/publisher/models/payload_segment.py index 92de802..df2e65e 100644 --- a/publisher/models/payload_segment.py +++ b/publisher/models/payload_segment.py @@ -2,11 +2,9 @@ """Segments of multi-part payloads, kept until the payload is complete.""" import datetime -import logging from typing import TYPE_CHECKING from sqlalchemy import ForeignKey, LargeBinary, select -from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Mapped, Session, mapped_column, relationship from publisher.db import Base @@ -15,8 +13,6 @@ if TYPE_CHECKING: from publisher.models import PayloadSession -logger = logging.getLogger(__name__) - class PayloadSegment(Base): __tablename__ = "payload_segments" @@ -36,23 +32,13 @@ class PayloadSegment(Base): ) -def create_if_not_exists( - session: Session, session_id: int, data: bytes -) -> PayloadSegment | None: - """Insert a segment; return None on a duplicate without poisoning the session.""" - try: - with session.begin_nested(): - seg = PayloadSegment(session_id=session_id, data=data) - session.add(seg) - session.flush() - return seg - except IntegrityError: - logger.warning("duplicate segment for session_id=%d, ignoring", session_id) - return None +def create(session: Session, session_id: int, data: bytes) -> None: + session.add(PayloadSegment(session_id=session_id, data=data)) + session.flush() def get_all_data(session: Session, session_id: int) -> list[bytes]: - """Return all segment data bytes for a session as a flat list.""" + """Return the data of every segment stored for a payload session.""" return list( session.scalars( select(PayloadSegment.data).where(PayloadSegment.session_id == session_id) diff --git a/publisher/models/payload_session.py b/publisher/models/payload_session.py index a1a577c..852f20a 100644 --- a/publisher/models/payload_session.py +++ b/publisher/models/payload_session.py @@ -2,9 +2,11 @@ """Multi-part payloads waiting for all their segments.""" import datetime +from collections.abc import Sequence from typing import TYPE_CHECKING from sqlalchemy import String, UniqueConstraint, select +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Mapped, Session, mapped_column, relationship from publisher.db import Base @@ -36,19 +38,20 @@ class PayloadSession(Base): ) -def create(session: Session, sender_id: str, session_id: int) -> PayloadSession: - payload_session = PayloadSession(sender_id=sender_id, session_id=session_id) - session.add(payload_session) - session.flush() - return payload_session - - -def get_by_sender_and_session( +def get_or_create_locked( session: Session, sender_id: str, session_id: int -) -> PayloadSession | None: - return session.scalar( - select(PayloadSession).filter_by(sender_id=sender_id, session_id=session_id) - ) +) -> PayloadSession: + """Return the payload session, inserting it if missing, locked until commit.""" + try: + with session.begin_nested(): + session.add(PayloadSession(sender_id=sender_id, session_id=session_id)) + except IntegrityError: + pass # It already exists. + return session.scalars( + select(PayloadSession) + .filter_by(sender_id=sender_id, session_id=session_id) + .with_for_update() + ).one() def delete(session: Session, payload_session: PayloadSession) -> None: @@ -57,12 +60,14 @@ def delete(session: Session, payload_session: PayloadSession) -> None: session.flush() -def delete_stale(session: Session, older_than: datetime.datetime) -> int: - """Delete payload sessions created before the cutoff.""" +def delete_stale( + session: Session, older_than: datetime.datetime +) -> Sequence[PayloadSession]: + """Delete and return payload sessions created before the cutoff.""" stale_sessions = session.scalars( select(PayloadSession).where(PayloadSession.created_at < older_than) ).all() for payload_session in stale_sessions: session.delete(payload_session) session.flush() - return len(stale_sessions) + return stale_sessions diff --git a/publisher/models/platform_adapter.py b/publisher/models/platform_adapter.py index 3f2b4c7..435ec4a 100644 --- a/publisher/models/platform_adapter.py +++ b/publisher/models/platform_adapter.py @@ -23,7 +23,6 @@ class PlatformAdapter(Base): # uuid5 of the source URL, which also names the adapter's directories. id: Mapped[str] = mapped_column(String(36), primary_key=True) source_url: Mapped[str] = mapped_column(String(255)) - # NULL for an adapter registered by `platforms import`. tag: Mapped[str | None] = mapped_column(String(100), default=None) commit: Mapped[str] = mapped_column(String(40)) name: Mapped[str] = mapped_column(String(100)) @@ -47,11 +46,10 @@ class PlatformAdapter(Base): __table_args__ = ( Index("uq_platform_adapters_name_proto_id", "name", "proto_id", unique=True), ) - # Every UPDATE and DELETE checks the version it loaded, so a concurrent change - # fails with StaleDataError instead of being overwritten. + # Every UPDATE and DELETE checks the version it loaded and raises + # StaleDataError if a concurrent change bumped it. __mapper_args__: ClassVar[dict] = {"version_id_col": version} - # Paths come from config, so moving the data directory needs no rewrite. @property def path(self) -> str: return str(PlatformsConfig.get().adapters_dir / self.id) @@ -77,7 +75,7 @@ def find( cat_id: int | None = None, include_disabled: bool = False, ) -> list[PlatformAdapter]: - """Enabled adapters only, unless include_disabled.""" + """Find adapters, enabled only unless include_disabled.""" stmt = select(PlatformAdapter).order_by( PlatformAdapter.name, PlatformAdapter.proto_id ) @@ -95,7 +93,11 @@ def find( def get_for_protocol( session: Session, platform: str, proto_id: int, *, include_disabled: bool = False ) -> PlatformAdapter: - """Raises NotImplementedError when no adapter serves platform over proto_id.""" + """Return the adapter serving platform over proto_id. + + Raises: + NotImplementedError: No adapter serves it. + """ adapters = find( session, name=platform, proto_id=proto_id, include_disabled=include_disabled ) diff --git a/publisher/models/platform_adapter_job.py b/publisher/models/platform_adapter_job.py index d5625a2..18f912f 100644 --- a/publisher/models/platform_adapter_job.py +++ b/publisher/models/platform_adapter_job.py @@ -13,7 +13,7 @@ from publisher.db.types import UTCDateTime, utc_now from publisher.errors import PublisherError -# Longer than the task's time limit, so only abandoned jobs are failed. +# Above platform_task.TIME_LIMIT_SECONDS: a job active this long was abandoned. STALE_AFTER = datetime.timedelta(hours=1) # Bytes, under MySQL's 65,535-byte TEXT limit. LOG_LIMIT = 60 * 1024 @@ -61,7 +61,11 @@ def create( from_commit: str | None = None, requested_by: uuid.UUID | None = None, ) -> PlatformAdapterJob: - """Raises JobBusyError while another job for the adapter is active.""" + """Queue a job for an adapter. + + Raises: + JobBusyError: While another job for the adapter is active. + """ job = PlatformAdapterJob( adapter_id=adapter_id, action=action, @@ -82,7 +86,7 @@ def create( def start(session: Session, job_id: uuid.UUID) -> PlatformAdapterJob | None: - """Mark a queued job running; None if it was already started or failed.""" + """Mark a queued job running, or return None if it was already started or failed.""" result = cast( CursorResult, session.execute( @@ -110,7 +114,7 @@ def finish( ) -> None: job = session.get_one(PlatformAdapterJob, job_id) job.state = state - # Cutting the bytes can split a character; the partial one is dropped. + # Keeps the last LOG_LIMIT bytes, dropping a character the cut splits. job.log = "\n".join(log).encode()[-LOG_LIMIT:].decode(errors="ignore") job.tag = tag or job.tag job.to_commit = to_commit @@ -159,8 +163,7 @@ def delete_older_than(session: Session, cutoff: datetime.datetime) -> int: result = cast( CursorResult, session.execute( - # By finished_at, so an old job that cleanup just failed as abandoned - # is kept for the full retention period. + # Retention counts from finished_at, not created_at. delete(PlatformAdapterJob).where(PlatformAdapterJob.finished_at < cutoff) ), ) diff --git a/publisher/models/publication_stats.py b/publisher/models/publication_stats.py index d1fd38c..fd14793 100644 --- a/publisher/models/publication_stats.py +++ b/publisher/models/publication_stats.py @@ -27,7 +27,7 @@ class PublicationStats(Base): failure_reason: Mapped[str | None] = mapped_column(String(255), default=None) created_at: Mapped[datetime.datetime] = mapped_column(UTCDateTime, default=utc_now) - # (created_at, id) serves keyset pagination; id breaks timestamp ties. + # (created_at, id) serves keyset pagination, with id breaking timestamp ties. __table_args__ = ( Index("ix_publication_stats_created_at_id", "created_at", "id"), Index("ix_publication_stats_status_created_at", "status", "created_at"), @@ -71,6 +71,7 @@ def record( platform_name: str | None = None, country_code: str | None = None, failure_reason: str | None = None, + created_at: datetime.datetime | None = None, ) -> PublicationStats: """Record the outcome of a publish attempt.""" stats = PublicationStats( @@ -79,6 +80,7 @@ def record( platform_name=platform_name, country_code=country_code, failure_reason=failure_reason, + created_at=created_at, ) session.add(stats) session.flush() diff --git a/publisher/models/server_ephemeral_key.py b/publisher/models/server_ephemeral_key.py index 2b3b449..aafdd6b 100644 --- a/publisher/models/server_ephemeral_key.py +++ b/publisher/models/server_ephemeral_key.py @@ -4,7 +4,7 @@ import datetime from typing import TYPE_CHECKING -from sqlalchemy import ForeignKey, Index, LargeBinary, UniqueConstraint +from sqlalchemy import ForeignKey, LargeBinary, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship from publisher.db import Base @@ -24,12 +24,10 @@ class ServerEphemeralKey(Base): key_index: Mapped[int] = mapped_column() private_key: Mapped[bytes] = mapped_column(PrivateEncryptedBinary()) public_key: Mapped[bytes] = mapped_column(LargeBinary(32)) - used: Mapped[bool] = mapped_column(default=False) created_at: Mapped[datetime.datetime] = mapped_column(default=utc_now) updated_at: Mapped[datetime.datetime] = mapped_column( default=utc_now, onupdate=utc_now ) - used_at: Mapped[datetime.datetime | None] = mapped_column(default=None) token_hash: Mapped["TokenHash"] = relationship( "TokenHash", back_populates="server_keys" @@ -39,5 +37,4 @@ class ServerEphemeralKey(Base): UniqueConstraint( "token_hash_id", "key_index", name="uq_server_keys_token_hash_id_key_index" ), - Index("ix_server_ephemeral_keys_token_hash_id_used", "token_hash_id", "used"), ) diff --git a/publisher/models/server_identity_key.py b/publisher/models/server_identity_key.py index 1e2acfa..a8a9596 100644 --- a/publisher/models/server_identity_key.py +++ b/publisher/models/server_identity_key.py @@ -14,8 +14,6 @@ class ServerIdentityKey(Base): - """Persistent x25519 server identity key.""" - __tablename__ = "server_identity_keys" id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) @@ -60,7 +58,7 @@ def get_public_key(session: Session, key_id: int) -> dict[str, Any]: def get_private_key(session: Session, key_id: int) -> X25519PrivateKey: - """Fetch a private key for cryptographic operations.""" + """Return the server identity private key at key_id.""" if not (0 <= key_id <= 255): raise ValueError(f"Invalid key_id {key_id}: must be 0-255") key = session.scalar( @@ -72,7 +70,7 @@ def get_private_key(session: Session, key_id: int) -> X25519PrivateKey: def mark_key_used(session: Session, key_id: int) -> None: - """Mark a server identity key as used after a successful operation.""" + """Record a use of a server identity key.""" if not (0 <= key_id <= 255): raise ValueError(f"Invalid key_id {key_id}: must be 0-255") result = cast( diff --git a/publisher/models/token_hash.py b/publisher/models/token_hash.py index e29c761..8ad98f1 100644 --- a/publisher/models/token_hash.py +++ b/publisher/models/token_hash.py @@ -36,8 +36,8 @@ class TokenHash(Base): "ClientEphemeralKey", back_populates="token_hash", cascade="all, delete-orphan" ) - # MySQL can't index a BLOB/TEXT column without an explicit key length; - # mysql_length is ignored on other dialects. + # mysql_length gives MySQL the key length it needs to index a BLOB column. + # Other dialects ignore it. __table_args__ = ( Index( "uq_token_hashes_token_hash", @@ -50,7 +50,7 @@ class TokenHash(Base): def create(session: Session, token_pk_id: int) -> tuple[TokenHash, bytes]: - """Create a token hash; return it with the raw token, which isn't stored.""" + """Create a token hash and return it with the raw token, which isn't stored.""" raw_token = secrets.token_bytes(32) token_hash_bytes = hashlib.sha256(raw_token).digest() diff --git a/publisher/platforms/adapter_config.py b/publisher/platforms/adapter_config.py index de1f93c..c3e3ced 100644 --- a/publisher/platforms/adapter_config.py +++ b/publisher/platforms/adapter_config.py @@ -73,7 +73,7 @@ def missing(adapter: PlatformAdapter) -> list[str]: def is_configured(adapter: PlatformAdapter) -> bool: - """Whether every required setting has a value; False if its files are broken.""" + """Whether every required setting has a value, or False if its files are broken.""" try: return not missing(adapter) except AdapterError as e: @@ -87,9 +87,9 @@ def update( values: dict[str, Any], actor: Credential | None = None, ) -> None: - """Change the settings given and keep the others; None removes a setting. + """Change the settings given and keep the others. None removes a setting. - The file is replaced before this returns; commit right after. + The file is replaced before this returns. Commit right after. Raises: InvalidConfigError: A key isn't a setting, or a value has the wrong type. @@ -107,8 +107,8 @@ def setup( ) -> None: """Run the adapter's setup and save the settings it returns, replacing the file. - End the transaction that loaded adapter first, since the adapter call can be - slow, and commit right after, as for update. + End the transaction that loaded adapter first, and commit right after, as for + update. Raises: InvalidConfigError: The inputs don't fit the adapter's [setup] tables, or @@ -119,7 +119,7 @@ def setup( """ declared = load_manifest(adapter) if not declared.setup: - raise AdapterError(f"{adapter.name} has no setup; set its config instead.") + raise AdapterError(f"{adapter.name} has no setup. Set its config instead.") inputs = _check(declared.setup, inputs, InvalidConfigError, require_all=True) try: result = ipc.call(adapter, "setup", SetupRequest(inputs)) @@ -163,8 +163,7 @@ def _save( actor: Credential | None, changed: dict[str, Any], ) -> None: - # Updating the row bumps its version, so a concurrent change to the adapter - # fails with a conflict. + # Touching the row makes a concurrent change to the adapter fail with a conflict. adapter.updated_at = utc_now() adapter.updated_by = actor.id if actor else None flush(session, adapter) diff --git a/publisher/platforms/ipc.py b/publisher/platforms/ipc.py index dab1302..14e2cf5 100644 --- a/publisher/platforms/ipc.py +++ b/publisher/platforms/ipc.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) TIMEOUT = 60 -# Adapters run third-party code, so they get only these from the environment. +# The only variables adapter processes get from the environment. PASSED_ENV = ( "PATH", "HOME", diff --git a/publisher/platforms/manager.py b/publisher/platforms/manager.py index 5ed8c27..4a4d25e 100644 --- a/publisher/platforms/manager.py +++ b/publisher/platforms/manager.py @@ -20,14 +20,12 @@ from publisher.config import PlatformsConfig from publisher.errors import PublisherError from publisher.models import audit_event -from publisher.models import platform_adapter as platform_adapters from publisher.models import token as tokens from publisher.models.audit_event import AuditAction from publisher.models.credential import Credential from publisher.models.platform_adapter import PROTOCOL_NAMES, PlatformAdapter from relaysms_adapter_sdk.manifest import Category, Manifest, ManifestError, Protocol from relaysms_adapter_sdk.manifest import load as load_manifest -from relaysms_adapter_sdk.paths import CREDENTIALS_FILE logger = logging.getLogger(__name__) @@ -165,12 +163,12 @@ def flush(session: Session, adapter: PlatformAdapter) -> None: ) from None except StaleDataError: raise AdapterConflictError( - f"Adapter {name!r} was changed concurrently; retry" + f"Adapter {name!r} was changed concurrently. Retry." ) from None def adapter_id(url: str) -> str: - """Stable per URL, so a reinstall finds the same directories.""" + """The adapter's id: a UUID derived from its source URL.""" return str(uuid.uuid5(uuid.NAMESPACE_URL, url.strip().lower())) @@ -242,12 +240,7 @@ def _register(session: Session, adapter: PlatformAdapter, manifest: Manifest) -> def activate(adapter: PlatformAdapter) -> None: - """Move a built version into place. Call it after the install or update commits. - - If it ran first and the commit failed, the new files would be in place while the - database still names the old version. - """ - # Renames are atomic, so an adapter call sees the old version or the new. + """Move a built version into place. Call it after the install or update commits.""" for current in (Path(adapter.path), Path(adapter.venv_path)): new, old = _beside(current, "new"), _beside(current, "old") if not new.exists(): @@ -312,8 +305,7 @@ def update( """Register tag, or the newest version tag, for an adapter.""" tag = tag or latest_tag(adapter.source_url) commit, manifest = _build(adapter, tag, log) - # Refused: the installed tag now points to other code, so it was moved after - # its release. + # Refuse the installed tag when it now points to a different commit. if tag == adapter.tag and commit != adapter.commit: _discard_build(adapter) raise AdapterError( @@ -356,17 +348,15 @@ def remove( *, force: bool = False, ) -> None: - """Unregister an adapter; call delete_files after the commit. + """Unregister an adapter. Call delete_files after the commit. Raises: AdapterInUseError: When accounts are linked through it and force is off. """ - # The id names its directories; reject one that could point elsewhere. + # Reject an id that could make its directory paths point elsewhere. if adapter.id in ("", ".", "..") or "/" in adapter.id: raise AdapterError(f"Unsafe adapter id {adapter.id!r}") linked = tokens.count_for_platform(session, adapter.name, adapter.proto_id) - # Refused while accounts are linked: without the adapter, their tokens can't - # be revoked at the platform. if linked and not force: raise AdapterInUseError( f"{linked} linked account(s) use {adapter.name!r}. Disable it instead, " @@ -385,66 +375,3 @@ def remove( ) session.delete(adapter) flush(session, adapter) - - -def import_from_disk(session: Session) -> list[PlatformAdapter]: - """Register adapter directories with no row, e.g. from the old JSON registry.""" - adapters_dir = PlatformsConfig.get().adapters_dir - if not adapters_dir.is_dir(): - return [] - - # Checked up front: one clash at flush would fail the whole import. - taken = { - (a.name, a.proto_id) - for a in platform_adapters.find(session, include_disabled=True) - } - imported = [] - for path in sorted(adapters_dir.iterdir()): - if not path.is_dir() or session.get(PlatformAdapter, path.name): - continue - try: - manifest = _read_manifest(path) - repo = Repo(path) - adapter = PlatformAdapter( - id=path.name, - source_url=repo.remotes.origin.url, - commit=repo.head.commit.hexsha, - ) - _apply_manifest(adapter, manifest) - except Exception as e: - logger.warning("Skipping adapter directory %s: %s", path, e) - continue - if (adapter.name, adapter.proto_id) in taken: - logger.warning("Skipping %s: %r is already installed", path, adapter.name) - continue - taken.add((adapter.name, adapter.proto_id)) - session.add(adapter) - imported.append(adapter) - session.flush() - return imported - - -def move_files_out_of_code(session: Session) -> list[PlatformAdapter]: - """Copy credentials.json to config dirs and move assets dirs to state dirs. - - Leaves an adapter alone where its new location already exists. - """ - legacy_state_dir = PlatformsConfig.get().adapters_state_dir.parent / "assets" - changed = [] - for adapter in platform_adapters.find(session, include_disabled=True): - credentials = Path(adapter.path) / CREDENTIALS_FILE - config = Path(adapter.config_path) / CREDENTIALS_FILE - legacy_state = legacy_state_dir / adapter.id - state = Path(adapter.state_path) - moved = False - if credentials.is_file() and not config.exists(): - config.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(credentials, config) - moved = True - if legacy_state.is_dir() and not state.exists(): - state.parent.mkdir(parents=True, exist_ok=True) - legacy_state.rename(state) - moved = True - if moved: - changed.append(adapter) - return changed diff --git a/publisher/publications.py b/publisher/publications.py index 721ec12..c895d24 100644 --- a/publisher/publications.py +++ b/publisher/publications.py @@ -19,11 +19,10 @@ from publisher.errors import PublisherError from publisher.keys import pop_token_keys from publisher.models import platform_adapter as platform_adapters -from publisher.models.payload_segment import create_if_not_exists as create_segment +from publisher.models.payload_segment import create as create_segment from publisher.models.payload_segment import get_all_data -from publisher.models.payload_session import create as create_session from publisher.models.payload_session import delete as delete_session -from publisher.models.payload_session import get_by_sender_and_session +from publisher.models.payload_session import get_or_create_locked from publisher.models.platform_adapter import PNBA, PlatformAdapter from publisher.models.server_identity_key import get_private_key, mark_key_used from publisher.models.token import Token, update_token_data @@ -84,14 +83,14 @@ def __init__( ): super().__init__(message, platform_name=platform_name) self.code = code - # A token the adapter refreshed before the send failed; it replaces the - # stored one. + # A token the adapter refreshed before the send failed, stored in place of + # the old one. self.token = token @dataclass(frozen=True) class Delivery: - """A decrypted payload ready to send; token_id is None for offline content.""" + """A decrypted payload ready to send, with no token_id for offline content.""" adapter: PlatformAdapter request: SendRequest @@ -169,10 +168,9 @@ def prepare( protocol: str | None = None, tag: str | None = None, ) -> Delivery | None: - """Decrypt a payload, deleting its keys; None while segments are missing. + """Decrypt a payload and delete its keys, or return None while segments are missing. - The deletion only sticks if the session commits, so commit it after a - successful send() and finish(), and roll it back otherwise. + Commit the session before calling send(). """ payload = _assemble( session, @@ -336,7 +334,7 @@ def _publish_online_content( def send(delivery: Delivery) -> dict | None: - """Send through the adapter; return the token it refreshed, if any. + """Send through the adapter and return the token it refreshed, if any. Raises: AdapterIntegrationError: The adapter failed. @@ -412,18 +410,18 @@ def _store_segment_and_try_join( ) -> rrs.V1Payloads | None: sess_id = rrs.v1_get_payload_session_id(payload_raw) - payload_session = get_by_sender_and_session(session, sender_id, sess_id) - if payload_session is None: - payload_session = create_session(session, sender_id, sess_id) - + payload_session = get_or_create_locked(session, sender_id, sess_id) create_segment(session, payload_session.id, raw_segment) - segment_data = get_all_data(session, payload_session.id) + # Drops segments delivered more than once. + segment_data = list(dict.fromkeys(get_all_data(session, payload_session.id))) try: joined = rrs.V1Payloads.join(segment_data) except (rrs.V1PayloadsError.NoLastSegments, rrs.V1PayloadsError.MissingSegments): logger.debug( - "Session %s waiting for segments, %d stored.", sess_id, len(segment_data) + "Payload session %s waiting for segments, %d stored.", + sess_id, + len(segment_data), ) return None except Exception as exc: @@ -436,7 +434,9 @@ def _store_segment_and_try_join( return None delete_session(session, payload_session) - logger.info("Session %s assembled from %d segments.", sess_id, len(segment_data)) + logger.info( + "Payload session %s assembled from %d segments.", sess_id, len(segment_data) + ) return joined diff --git a/publisher/smtp/auth.py b/publisher/smtp/auth.py index 8e90955..74e1ba0 100644 --- a/publisher/smtp/auth.py +++ b/publisher/smtp/auth.py @@ -31,8 +31,7 @@ def is_sender_allowed(email_address: str) -> bool: def _trusted_result(msg: Message) -> authres.core.AuthenticationResultsHeader | None: """First Authentication-Results header matching SMTP_TRUSTED_AUTHSERV_ID, if any. - Headers from any other (or missing) authserv-id are ignored, since a - sender can put arbitrary text of their own in this header. + Headers from any other (or missing) authserv-id are ignored. """ trusted_id = smtp_config.trusted_authserv_id if not trusted_id: @@ -53,7 +52,7 @@ def _trusted_result(msg: Message) -> authres.core.AuthenticationResultsHeader | def evaluate_authentication(msg: Message) -> tuple[bool, str]: """Check SPF/DKIM verdicts from a trusted Authentication-Results header.""" if not smtp_config.trusted_authserv_id: - return False, "SMTP_TRUSTED_AUTHSERV_ID not configured; rejecting all mail" + return False, "SMTP_TRUSTED_AUTHSERV_ID not configured, rejecting all mail" header = _trusted_result(msg) if header is None: diff --git a/publisher/smtp/listener.py b/publisher/smtp/listener.py index c884efc..94a0063 100644 --- a/publisher/smtp/listener.py +++ b/publisher/smtp/listener.py @@ -45,7 +45,7 @@ def _ping_heartbeat() -> None: try: requests.get(url, timeout=5) except requests.RequestException as exc: - logger.warning("Failed to ping SMTP listener heartbeat: %s", exc) + logger.warning("Failed to ping SMTP listener heartbeat: %s", type(exc).__name__) def _mask_email(address: str) -> str: @@ -58,9 +58,8 @@ def _mask_email(address: str) -> str: def process_incoming_email(msg: MailMessage) -> bool: """Validate, authenticate, and queue an incoming email for publication. - Returns whether the email is done with (should be deleted). False means - an unexpected error occurred and it should be left for a retry next - cycle; every other outcome, handled or intentionally rejected, is done. + Return True when the email can be deleted, or False after an unexpected error, + to retry it next cycle. """ email_uid = msg.uid from_email = msg.from_ @@ -72,20 +71,18 @@ def process_incoming_email(msg: MailMessage) -> bool: if not smtp_auth.is_sender_allowed(from_email): logger.warning( - "Dropping email %s from unauthorized sender: %s", + "Discarding email %s from unauthorized sender: %s", email_uid, _mask_email(from_email), ) return True - # as_bytes() re-serializes rather than returning the literal wire - # bytes, but dkimpy tolerates that (see auth.verify_dkim_independently). auth_passed, auth_reason = smtp_auth.evaluate( msg.obj, msg.obj.as_bytes(), from_email ) if not auth_passed: logger.warning( - "Dropping email %s failing authentication (sender %s): %s", + "Discarding email %s failing authentication (sender %s): %s", email_uid, _mask_email(from_email), auth_reason, @@ -197,7 +194,7 @@ def main() -> None: socket.herror, socket.gaierror, ) as e: - logger.error("Error occurred: %s", e) + logger.error("Mailbox connection failed: %s", e) logger.error(traceback.format_exc()) logger.info("Reconnecting in a minute...") time.sleep(60) diff --git a/publisher/tasks/celery_app.py b/publisher/tasks/celery_app.py index 063b167..0c865b4 100644 --- a/publisher/tasks/celery_app.py +++ b/publisher/tasks/celery_app.py @@ -80,7 +80,6 @@ def make_celery() -> Celery: }, } if celery.worker_heartbeat_url: - # Uptime Kuma push-monitor heartbeat, see docs/operating/observability.md beat_schedule["worker-heartbeat"] = { "task": "tasks.heartbeat_task.ping_worker_heartbeat", "schedule": 60.0, diff --git a/publisher/tasks/cleanup_task.py b/publisher/tasks/cleanup_task.py index 6fafce6..24bd2fa 100644 --- a/publisher/tasks/cleanup_task.py +++ b/publisher/tasks/cleanup_task.py @@ -13,6 +13,7 @@ delete_expired as delete_expired_credential_sessions, ) from publisher.models.payload_session import delete_stale +from publisher.models.publication_stats import record as record_publication from publisher.tasks.celery_app import celery_app logger = logging.getLogger(__name__) @@ -21,20 +22,27 @@ @celery_app.task(name="tasks.cleanup_task.cleanup_stale_payload_sessions") def cleanup_stale_payload_sessions() -> None: - """Delete payload sessions left incomplete for longer than the max age.""" + """Delete payload sessions incomplete past the max age, recording each as failed.""" cutoff = utc_now() - cleanup_config.payload_session_max_age with get_session() as db: - deleted = delete_stale(db, cutoff) - - if deleted: - logger.info("Cleaned up %d stale payload session(s)", deleted) + stale = delete_stale(db, cutoff) + for payload_session in stale: + record_publication( + db, + status="failed", + failure_reason="segments_missing", + created_at=payload_session.created_at, + ) + + if stale: + logger.info("Cleaned up %d stale payload session(s)", len(stale)) else: logger.debug("No stale payload sessions to clean up") @celery_app.task(name="tasks.cleanup_task.cleanup_idle_tokens") def cleanup_idle_tokens() -> None: - """Delete tokens (and their keys) idle past the configured max age.""" + """Delete idle tokens and their keys, and revoke them upstream.""" cutoff = utc_now() - cleanup_config.token_idle_max_age with get_session() as db: deleted = tokens.delete_idle(db, cutoff) diff --git a/publisher/tasks/heartbeat_task.py b/publisher/tasks/heartbeat_task.py index 7cde704..fee2860 100644 --- a/publisher/tasks/heartbeat_task.py +++ b/publisher/tasks/heartbeat_task.py @@ -13,7 +13,7 @@ @celery_app.task(name="tasks.heartbeat_task.ping_worker_heartbeat") def ping_worker_heartbeat() -> None: - """Ping the Uptime Kuma push monitor. No-op if unconfigured.""" + """GET UPTIME_KUMA_WORKER_PUSH_URL, if set.""" url = celery_config.worker_heartbeat_url if not url: return @@ -21,4 +21,4 @@ def ping_worker_heartbeat() -> None: try: requests.get(url, timeout=5) except requests.RequestException as exc: - logger.warning("Failed to ping worker heartbeat: %s", exc) + logger.warning("Failed to ping worker heartbeat: %s", type(exc).__name__) diff --git a/publisher/tasks/platform_task.py b/publisher/tasks/platform_task.py index c337ccf..9048e2f 100644 --- a/publisher/tasks/platform_task.py +++ b/publisher/tasks/platform_task.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -# Under jobs.STALE_AFTER, so cleanup never fails a job that's still running. +# Must stay under jobs.STALE_AFTER. TIME_LIMIT_SECONDS = 30 * 60 @@ -52,7 +52,7 @@ def run_adapter_job(job_id: str) -> None: except PublisherError as e: log.append(f"Failed: {e}") except Exception as e: - logger.exception("Adapter job %s failed unexpectedly", job_id) + logger.exception("Unexpected error in adapter job %s", job_id) log.append(f"Failed unexpectedly: {e}") with get_session() as db: diff --git a/publisher/tasks/publication_task.py b/publisher/tasks/publication_task.py index 2ff20d8..3efe717 100644 --- a/publisher/tasks/publication_task.py +++ b/publisher/tasks/publication_task.py @@ -34,8 +34,7 @@ logger = logging.getLogger(__name__) -# Stats keep fixed codes so they group cleanly. Error messages stay in the logs: -# adapters pass on upstream text that can name recipients or echo responses. +# Fixed failure codes for the stats. Error messages go only to the logs. _FAILURE_REASONS = { PayloadMalformedError: "payload_malformed", PayloadNotSupportedError: "payload_not_supported", @@ -59,7 +58,7 @@ def _failure_reason(exc: Exception) -> str: if isinstance(exc, AdapterIntegrationError): return _ADAPTER_FAILURE_REASONS.get(exc.code, "adapter_error") - # The class hierarchy, so a subclass of a mapped error keeps its code. + # A subclass of a mapped error gets its base class's code. for cls in type(exc).__mro__: if cls in _FAILURE_REASONS: return _FAILURE_REASONS[cls] @@ -81,10 +80,8 @@ def publish_message( ) -> None: """Validate and publish a payload, then record the outcome.""" delivery = refreshed_token = None + status = "failed" try: - # Commits only after a successful send. A failure rolls back, so the payload's - # keys can be used again, and a duplicate that arrives during the send waits - # on their row lock. with get_session() as db: payload_raw, raw_segment, payload_type = publications.validate(text_payload) delivery = publications.prepare( @@ -96,26 +93,15 @@ def publish_message( protocol=protocol, tag=tag, ) - if delivery is None: - # Segments of a multi-part payload are still missing. - return - new_token = publications.send(delivery) - publications.finish(db, delivery, new_token) - record_publication( - db, - protocol=protocol, - status="published", - platform_name=delivery.platform, - country_code=country_code, - ) + if delivery is None: + # Segments of a multi-part payload are still missing. return - except ( - PayloadMalformedError, - PayloadNotSupportedError, - ProtocolNotAllowedError, - OfflineTagError, - KeyManagementError, - ) as exc: + new_token = publications.send(delivery) + except (ProtocolNotAllowedError, OfflineTagError) as exc: + logger.warning("Rejected payload: %s", exc) + status = "rejected" + platform_name, failure_reason = exc.platform_name, _failure_reason(exc) + except (PayloadMalformedError, PayloadNotSupportedError, KeyManagementError) as exc: logger.error("Failed to process payload: %s", exc) platform_name, failure_reason = exc.platform_name, _failure_reason(exc) except AdapterIntegrationError as exc: @@ -126,6 +112,22 @@ def publish_message( logger.exception("An unexpected error occurred during task processing.") platform_name = delivery.platform if delivery else None failure_reason = _failure_reason(exc) + else: + try: + with get_session() as db: + publications.finish(db, delivery, new_token) + record_publication( + db, + protocol=protocol, + status="published", + platform_name=delivery.platform, + country_code=country_code, + ) + except Exception: + logger.exception( + "Sent via %r, but couldn't record the result.", delivery.platform + ) + return with get_session() as db: if delivery and refreshed_token: @@ -133,7 +135,7 @@ def publish_message( record_publication( db, protocol=protocol, - status="failed", + status=status, platform_name=platform_name, country_code=country_code, failure_reason=failure_reason, diff --git a/publisher/tokens.py b/publisher/tokens.py index 2c93e68..dc090de 100644 --- a/publisher/tokens.py +++ b/publisher/tokens.py @@ -26,19 +26,18 @@ def account(token: Token) -> Account: def revocation(session: Session, token: Token) -> Revocation | None: - """Return the adapter and request that revoke a token; call it before deleting it. + """Return the adapter and request that revoke a token. Call it before deleting it. None when no adapter serves its platform. Pass the result to revoke_upstream after the deletion commits. """ try: - # Disabled adapters still revoke, so users can always unlink. adapter = platform_adapters.get_for_protocol( session, token.platform, token.proto_id, include_disabled=True ) except NotImplementedError: logger.warning( - "No adapter for %r; token %d can't be revoked upstream.", + "No adapter for %r: token %d can't be revoked upstream.", token.platform, token.token_id, ) @@ -47,7 +46,7 @@ def revocation(session: Session, token: Token) -> Revocation | None: def revoke_upstream(revocation: Revocation | None) -> None: - """Revoke a token at the platform once its deletion commits; failures are logged.""" + """Revoke a token at the platform after its deletion commits, logging failures.""" if revocation is None: return adapter, request = revocation @@ -62,7 +61,7 @@ def revoke_upstream(revocation: Revocation | None) -> None: def delete_idle( session: Session, older_than: datetime.datetime ) -> list[tuple[str, Revocation | None]]: - """Delete tokens unused since older_than; return their platforms and revocations.""" + """Delete tokens unused since older_than, returning platforms and revocations.""" deleted = [] for token in get_idle(session, older_than): deleted.append((token.platform, revocation(session, token))) diff --git a/pyproject.toml b/pyproject.toml index 0a16228..a473e32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ select = [ "D", "PLR6301", ] -# Docstrings are only written where they add something, so none are required. +# Docstrings are optional. ignore = ["D1"] [tool.ruff.lint.pydocstyle] @@ -69,7 +69,7 @@ exclude = [ root = "sdk" extraPaths = ["sdk/src", "sdk/examples/echo-adapter/src"] -# A None where a test expects a value fails the test at runtime anyway. +# Tests use Optional values without narrowing them first. [[tool.pyright.executionEnvironments]] root = "tests" extraPaths = ["."] @@ -78,7 +78,7 @@ reportOperatorIssue = false [tool.pytest.ini_options] testpaths = ["tests"] -# Adds a results table to the GitHub Actions job summary; idle elsewhere. +# Adds a results table to the GitHub Actions job summary when run there. addopts = ["-p", "tests.summary"] pythonpath = ["."] # make test-dialects and make test-e2e run these on their own. diff --git a/requirements-dev.txt b/requirements-dev.txt index 36a59cd..36ee718 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,4 @@ pre-commit==4.7.0 pyright==1.1.414 pytest==9.1.1 pytest-cov==7.1.0 -ruff==0.17.1 +ruff==0.17.2 diff --git a/requirements.txt b/requirements.txt index 32d784f..5822b08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,18 @@ -e ./sdk -alembic==1.20.0 +alembic==1.20.1 argon2-cffi==25.1.0 authres==1.2.0 -cachetools==7.2.3 +cachetools==7.2.5 celery[sqlalchemy,redis]==5.6.3 cryptography==50.0.2 dkimpy==1.1.8 -fastapi==0.149.0 +fastapi==0.162.0 GitPython==3.2.1 grpc-interceptor==0.15.4 -grpcio==1.84.0 -grpcio-health-checking==1.84.0 -grpcio-testing==1.84.0 -grpcio-tools==1.84.0 +grpcio==1.85.0 +grpcio-health-checking==1.85.0 +grpcio-testing==1.85.0 +grpcio-tools==1.85.0 imap-tools==1.16.0 msgspec==0.22.0 opentelemetry-api==1.45.1 diff --git a/scripts/build-book.sh b/scripts/build-book.sh index 17cf288..a7408f4 100755 --- a/scripts/build-book.sh +++ b/scripts/build-book.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # SPDX-License-Identifier: GPL-3.0-only -# Builds the manual in docs/ into book/ with pinned mdbook and mdbook-mermaid -# releases, downloaded once into .mdbook/. +# Builds the manual in docs/ into book/ with pinned mdbook, mdbook-mermaid and +# Mermaid releases, downloaded once into .mdbook/. set -Eeuo pipefail @@ -14,7 +14,11 @@ TOOLS_DIR="$ROOT_DIR/.mdbook" SERVE=false MDBOOK_VERSION=0.5.3 -MERMAID_VERSION=0.17.1 +MDBOOK_MERMAID_VERSION=0.17.1 +# mermaid.min.js release that replaces the older one mdbook-mermaid bundles. +MERMAID_JS_VERSION=12.2.0 +MERMAID_JS_SHA256=a8a6a6f805b074a8261a2825c86f99c8054acb3b35f50bb7e24a4f9f2c3c3b9f +MERMAID_JS="$TOOLS_DIR/mermaid-$MERMAID_JS_VERSION.min.js" declare -A SHA256=( ["mdbook-x86_64"]=dfb86d8e20a3fed91bf549aa450b9d7f46a275b700292e050cdfd3171732e7fd ["mdbook-aarch64"]=428c91cf73fec7494cbefe6e170e12508c9d45d9d096104b7c62d47ff43911c2 @@ -69,15 +73,31 @@ fetch_tool() { log "Installed $tool v$version" } +fetch_mermaid_js() { + [ -f "$MERMAID_JS" ] && return + mkdir -p "$TOOLS_DIR" + # Unlike fetch_tool's mktemp file (mode 0600), this file is published in the book. + curl -fsSL -o "$MERMAID_JS.tmp" \ + "https://cdn.jsdelivr.net/npm/mermaid@$MERMAID_JS_VERSION/dist/mermaid.min.js" + echo "$MERMAID_JS_SHA256 $MERMAID_JS.tmp" | sha256sum --check --quiet || { + rm -f "$MERMAID_JS.tmp" + error "mermaid.min.js v$MERMAID_JS_VERSION download doesn't match its sha256" + } + mv "$MERMAID_JS.tmp" "$MERMAID_JS" + log "Installed mermaid.min.js v$MERMAID_JS_VERSION" +} + main() { parse_args "$@" fetch_tool mdbook "$MDBOOK_VERSION" rust-lang/mdBook - fetch_tool mdbook-mermaid "$MERMAID_VERSION" badboy/mdbook-mermaid - export PATH="$TOOLS_DIR/mdbook-$MDBOOK_VERSION:$TOOLS_DIR/mdbook-mermaid-$MERMAID_VERSION:$PATH" + fetch_tool mdbook-mermaid "$MDBOOK_MERMAID_VERSION" badboy/mdbook-mermaid + fetch_mermaid_js + export PATH="$TOOLS_DIR/mdbook-$MDBOOK_VERSION:$TOOLS_DIR/mdbook-mermaid-$MDBOOK_MERMAID_VERSION:$PATH" cd "$ROOT_DIR" - # Writes mermaid.min.js and mermaid-init.js from the pinned release. + # Writes mermaid-init.js and the bundled mermaid.min.js. RUST_LOG=warn mdbook-mermaid install . >/dev/null + cp "$MERMAID_JS" mermaid.min.js if [ "$SERVE" = true ]; then mdbook serve else diff --git a/scripts/lib.sh b/scripts/lib.sh index 1690217..3e7a65d 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -2,7 +2,6 @@ # SPDX-License-Identifier: GPL-3.0-only # Shared helpers sourced by the other scripts in this repo. -# Colors are skipped when the relevant stream isn't a terminal. _no_color="${NO_COLOR:-}" if [[ -t 1 && -z "$_no_color" ]]; then _grn=$'\033[0;32m' @@ -17,8 +16,7 @@ else _ylw='' fi -# $1=color $2=symbol $3=fd (1 or 2) $4=message. Reset code is derived from -# whether $1 is set, so callers don't have to track a matching "off" value. +# $1=color $2=symbol $3=fd (1 or 2) $4=message. _log_line() { local now off='' printf -v now '%(%Y-%m-%d %H:%M:%S)T' -1 @@ -35,7 +33,7 @@ error() { on_err() { _log_line "$_red" 'x' 2 "ERROR: aborted at line $1 (last command: $2)"; } trap 'on_err "$LINENO" "$BASH_COMMAND"' ERR -# Keeps output like generated credentials from getting lost in the log. +# Prints the lines in a banner, e.g. generated credentials. highlight() { local line echo @@ -47,22 +45,18 @@ highlight() { echo } -# Excludes characters unsafe in SQL/AMQP, and can't start with - (CLI flag). validate_identifier() { local name="$1" value="$2" [[ "$value" =~ ^[A-Za-z_][A-Za-z0-9_]{0,63}$ ]] || error "$name must contain only letters, digits, and underscores, and start with a letter or underscore (got: '$value')" } -# Excludes characters that could break out of SQL/sed/amqp:// contexts. validate_secret() { local name="$1" value="$2" [[ "$value" =~ ^[A-Za-z0-9_.,!?+=~^-]+$ ]] || error "$name contains unsupported characters (letters, digits, and _.,!?+=~^- only)" } -# Rejects slashes (path traversal into nginx conf paths) and a leading - -# (could be mistaken for a certbot flag). validate_hostname() { local name="$1" value="$2" [[ "$value" =~ ^[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?(\.[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?)+$ ]] || @@ -73,7 +67,7 @@ validate_hostname() { # bash` (stdin is the script itself there). Falls back to $default if no # tty is reachable. prompt() { - # __-prefixed so a caller's result variable named e.g. reply isn't shadowed. + # The __ prefix keeps these locals from shadowing the caller's result variable. local __resultvar="$1" __question="> $2" __default="${3:-}" __reply="" if [ -t 0 ]; then read -r -p "$__question" __reply @@ -141,9 +135,7 @@ set_unit_names() { render_units() { local unit_user="$1" rw_paths rw_paths="$(app_directories | paste -sd ' ')" - # Only matches unit-name references (PartOf=, WantedBy=, ...), never - # Description=/Documentation=: those read "RelaySMS Publisher" (space, - # capitalized), not this lowercase-hyphenated pattern. + # Renames the unit references (PartOf=, WantedBy=, ...) for the instance. local instance_sed_args=() template name if [ -n "${INSTANCE_NAME:-}" ]; then instance_sed_args+=(-e "s/relaysms-publisher\.target/$(unit_name_for "$TARGET_UNIT_TEMPLATE")/g") @@ -171,7 +163,7 @@ read_instance_name() { [ -f "$INSTALL_DIR/.instance-name" ] && cat "$INSTALL_DIR/.instance-name" || true } -# `|| true` on the grep stops a no-match from tripping pipefail. +# Prints the last value of $1 in env file $2, without quotes. read_env_var() { local key="$1" file="$2" val val=$( (grep -E "^(export[[:space:]]+)?${key}[[:space:]]*=" "$file" 2>/dev/null || true) | @@ -184,8 +176,8 @@ read_env_var() { } # Every directory the services write to, one per line, absolute and -# deduplicated. Vars unset in .env fall back to template.env, which holds the -# same defaults as publisher/config.py. Expects INSTALL_DIR to already be set. +# deduplicated. Vars unset in .env fall back to template.env. Expects +# INSTALL_DIR to already be set. app_directories() { local var kind path while read -r var kind; do @@ -214,8 +206,6 @@ ensure_app_directories() { root="$(realpath "$INSTALL_DIR")" while read -r dir; do [ "$dir" != "$INSTALL_DIR" ] || continue - # The services can write these dirs, so a symlink planted in one could - # point root's chown at any directory. if [ "$(realpath -m "$dir")" != "${dir/#"$INSTALL_DIR"/$root}" ]; then warn "Skipping $dir: its path goes through a symlink" continue @@ -227,6 +217,16 @@ ensure_app_directories() { done < <(app_directories) } +run_config_check() { + log "Checking configuration" + (cd "$INSTALL_DIR" && sudo -u "$SERVICE_USER" venv/bin/python -m publisher config check) +} + +run_migrations() { + log "Running database migrations" + (cd "$INSTALL_DIR" && sudo -u "$SERVICE_USER" venv/bin/python -m alembic upgrade head) +} + # Prints nothing when the services aren't installed. Expects INSTANCE_NAME to # already be set by the caller. installed_service_user() { @@ -291,8 +291,8 @@ request_certificate() { certbot "${args[@]}" } -# gRPC needs HTTP/2, which certbot never enables. nginx 1.25.1+ takes -# "http2 on;"; below 1.25.1 only "listen ... http2" works. +# Enables HTTP/2 for gRPC in site file $1: "http2 on;" on nginx 1.25.1+, +# "listen ... http2" below. enable_nginx_http2() { local conf="$1" version version=$(nginx -v 2>&1 | sed -n 's#.*nginx/\([0-9.]*\).*#\1#p') diff --git a/scripts/migrate-runtime-data.sh b/scripts/migrate-runtime-data.sh deleted file mode 100755 index 3030c03..0000000 --- a/scripts/migrate-runtime-data.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: GPL-3.0-only -# Moves runtime data from platforms/ and gateway_clients/ into data/. Safe to -# rerun: paths already moved, or set in .env to somewhere else, are left alone. -# Run with the services stopped, as root or as the install's owner. - -set -Eeuo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=scripts/lib.sh -source "$SCRIPT_DIR/lib.sh" - -INSTALL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -ENV_FILE="$INSTALL_DIR/.env" - -# Env var, old default, new default (both relative to INSTALL_DIR). -MOVES=( - "PLATFORMS_ADAPTERS_DIR platforms/adapters data/platforms/adapters" - "PLATFORMS_ADAPTERS_VENV_DIR platforms/adapters_venv data/platforms/venvs" - "PLATFORMS_ADAPTERS_STATE_DIR platforms/adapters_assets data/platforms/state" - "PLATFORMS_ADAPTERS_STATE_DIR data/platforms/assets data/platforms/state" - "GATEWAY_CLIENTS_REGISTRY_FILE gateway_clients/registry.json data/gateway_clients/registry.json" -) - -move_one() { - local var="$1" old="$2" new="$3" value="" - [ -f "$ENV_FILE" ] && value="$(read_env_var "$var" "$ENV_FILE")" - # A custom location stays where it is. - case "$value" in - "" | "$old" | "$INSTALL_DIR/$old") ;; - *) return ;; - esac - - if [ -e "$INSTALL_DIR/$old" ] && [ ! -e "$INSTALL_DIR/$new" ]; then - mkdir -p "$(dirname "$INSTALL_DIR/$new")" - mv "$INSTALL_DIR/$old" "$INSTALL_DIR/$new" - log "Moved $old to $new" - fi - - if [ -n "$value" ]; then - local new_value="$new" - [[ "$value" = /* ]] && new_value="$INSTALL_DIR/$new" - sed -i "s#^$var=.*#$var=$new_value#" "$ENV_FILE" - log "Set $var=$new_value in .env" - fi -} - -# Old=new path pairs, absolute and relative, for every move that has happened. -moved_paths() { - local entry var old new - for entry in "${MOVES[@]}"; do - read -r var old new <<<"$entry" - [ -e "$INSTALL_DIR/$new" ] && [ ! -e "$INSTALL_DIR/$old" ] || continue - echo "$INSTALL_DIR/$old=$INSTALL_DIR/$new" - echo "$old=$new" - done -} - -# venv scripts hardcode their venv's absolute path in the shebang. -rewrite_venv_scripts() { - local venvs="$INSTALL_DIR/data/platforms/venvs" rename old new - [ -d "$venvs" ] || return 0 - while read -r rename; do - old="${rename%%=*}" new="${rename#*=}" - [[ "$old" = /* ]] || continue - { grep -rlIF "$old/" "$venvs" 2>/dev/null || true; } | - while read -r file; do sed -i "s#$old/#$new/#g" "$file"; done - done < <(moved_paths) -} - -# Migration 019 logs the overrides it finds in data/gateway_clients/, since they -# are no longer used, so copy any local entries there. -move_overrides() { - local old="$INSTALL_DIR/gateway_clients/mcc_mnc_overrides.json" - local new="$INSTALL_DIR/data/gateway_clients/mcc_mnc_overrides.json" - [ -f "$old" ] && [ ! -e "$new" ] || return 0 - if [ "$(tr -d '[:space:]' <"$old")" != "[]" ]; then - mkdir -p "$(dirname "$new")" - cp -p "$old" "$new" - log "Moved MCC/MNC overrides to data/gateway_clients/" - fi -} - -rename_assets_var() { - [ -f "$ENV_FILE" ] && grep -q '^PLATFORMS_ADAPTERS_ASSETS_DIR=' "$ENV_FILE" || return 0 - sed -i 's#^PLATFORMS_ADAPTERS_ASSETS_DIR=#PLATFORMS_ADAPTERS_STATE_DIR=#' "$ENV_FILE" - log "Renamed PLATFORMS_ADAPTERS_ASSETS_DIR to PLATFORMS_ADAPTERS_STATE_DIR in .env" -} - -main() { - local entry - rename_assets_var - for entry in "${MOVES[@]}"; do - # shellcheck disable=SC2086 # entries are space-separated fields - move_one $entry - done - rewrite_venv_scripts - move_overrides -} - -main "$@" diff --git a/scripts/run.sh b/scripts/run.sh index 896464a..2a01efa 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -6,7 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=scripts/lib.sh source "$SCRIPT_DIR/lib.sh" -# Export .env so the host, port and OTEL settings used below match the app's. +# Exports .env for the host, port and OTEL settings below. ENV_FILE="$SCRIPT_DIR/../.env" if [ -f "$ENV_FILE" ]; then set -a @@ -21,7 +21,7 @@ PORT="${PORT:-16000}" WORKERS="${WORKERS:-1}" OTEL_WRAP="$SCRIPT_DIR/otel-wrap.sh" -log "Starting gRPC, FastAPI, Celery worker, Celery beat scheduler ..." +log "Starting gRPC, REST, Celery worker and beat ..." OTEL_SERVICE_NAME="relaysms-publisher-grpc" "$OTEL_WRAP" "$PYTHON" -u -m publisher.api.grpc & GRPC_PID=$! diff --git a/scripts/setup-mysql.sh b/scripts/setup-mysql.sh index 7b579f4..017b5e5 100755 --- a/scripts/setup-mysql.sh +++ b/scripts/setup-mysql.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # SPDX-License-Identifier: GPL-3.0-only -# Re-runnable: an existing install or database/user is left as-is. +# Re-runnable: keeps an existing server, database and user, and sets the user's +# password to this run's. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -76,7 +77,7 @@ parse_args() { main() { parse_args "$@" require_root - [ -f "$INSTALL_DIR/.env" ] || error "$INSTALL_DIR/.env not found, run install.sh first" + [ -f "$INSTALL_DIR/.env" ] || error "$INSTALL_DIR/.env not found. Run install.sh first." validate_identifier "--db-name" "$DB_NAME" validate_identifier "--db-user" "$DB_USER" if [ "$DB_EXISTING" != "1" ] && { [ "$DB_HOST" != "127.0.0.1" ] || [ "$DB_PORT" != "3306" ]; }; then diff --git a/scripts/setup-observability.sh b/scripts/setup-observability.sh index 42a7d70..a23b4b5 100755 --- a/scripts/setup-observability.sh +++ b/scripts/setup-observability.sh @@ -131,7 +131,7 @@ container_running() { } # Loops until the port named by $1 is free or the user opts out (returns -# 1). Non-remappable ports just get re-checked after the user frees them. +# 1). With $3=0 the port can't change and is re-checked after the user frees it. resolve_port() { local __resultvar="$1" label="$2" remappable="$3" port="${!1}" ack="" while ! port_is_free "$port"; do @@ -177,7 +177,7 @@ setup_reverse_proxy() { main() { parse_args "$@" require_root - [ -d "$INSTALL_DIR/observability" ] || error "$INSTALL_DIR/observability not found, run install.sh first" + [ -d "$INSTALL_DIR/observability" ] || error "$INSTALL_DIR/observability not found. Run install.sh first." [ -z "$SITE_NAME" ] || validate_hostname "--site-name" "$SITE_NAME" [ -z "$KUMA_SITE_NAME" ] || validate_hostname "--kuma-site-name" "$KUMA_SITE_NAME" [ -n "$SITE_NAME" ] && [ -n "$KUMA_SITE_NAME" ] && [ "$SITE_NAME" = "$KUMA_SITE_NAME" ] && @@ -198,7 +198,7 @@ main() { systemctl enable docker &>/dev/null || true systemctl start docker docker compose version &>/dev/null || - error "docker compose plugin not available; install docker-compose-plugin manually" + error "docker compose plugin not available. Install docker-compose-plugin manually." SIGNOZ_UI_RUNNING=0 container_running relaysms-publisher-signoz-signoz-0 && SIGNOZ_UI_RUNNING=1 @@ -240,16 +240,13 @@ main() { if ! command -v foundryctl &>/dev/null; then log "Installing foundryctl" - # ~/.local/bin (foundryctl's default) isn't reliably on PATH for a - # non-interactive root shell. curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_INSTALL_DIR=/usr/local/bin bash fi signoz_external_url="http://localhost:$SIGNOZ_PORT" [ -n "$SITE_NAME" ] && [ "$SKIP_NGINX" != "1" ] && signoz_external_url="https://$SITE_NAME" - # Generated once; a rerun with a different --signoz-port after this file - # exists won't change it, same as the Postgres credentials. + # Generated on the first run only. if [ ! -f observability/signoz/casting.yaml ]; then log "Generating SigNoz configuration" pg_password=$(openssl rand -hex 24) @@ -281,9 +278,8 @@ main() { up -d fi - # Until an org exists, SigNoz's opamp bug (casting.yaml.template) keeps - # the OTLP receiver closed. Safe to call every run: setupCompleted flips - # permanently once an org exists, and a second registration is rejected. + # SigNoz keeps the OTLP receiver closed until an org exists, which + # registering the admin account creates. log "Waiting for the SigNoz API" signoz_version="" for _ in $(seq 1 60); do @@ -326,7 +322,7 @@ main() { log "Uptime Kuma already running, skipping" else log "Starting Uptime Kuma" - # Compose reads this .env, so a manual `up -d` later keeps the port. + # Compose reads KUMA_PORT from this .env. echo "KUMA_PORT=$KUMA_PORT" >observability/uptime-kuma/.env docker compose -f observability/uptime-kuma/docker-compose.yml up -d fi diff --git a/scripts/setup-postgres.sh b/scripts/setup-postgres.sh index f9fc93b..f23a2b1 100755 --- a/scripts/setup-postgres.sh +++ b/scripts/setup-postgres.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # SPDX-License-Identifier: GPL-3.0-only -# Re-runnable: an existing install or database/role is left as-is. +# Re-runnable: keeps an existing server, database and role, and sets the role's +# password and the database owner to this run's. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -76,7 +77,7 @@ parse_args() { main() { parse_args "$@" require_root - [ -f "$INSTALL_DIR/.env" ] || error "$INSTALL_DIR/.env not found, run install.sh first" + [ -f "$INSTALL_DIR/.env" ] || error "$INSTALL_DIR/.env not found. Run install.sh first." validate_identifier "--db-name" "$DB_NAME" validate_identifier "--db-user" "$DB_USER" if [ "$DB_EXISTING" != "1" ] && { [ "$DB_HOST" != "127.0.0.1" ] || [ "$DB_PORT" != "5432" ]; }; then @@ -123,8 +124,7 @@ END sudo -u postgres psql -v ON_ERROR_STOP=1 -c "ALTER DATABASE \"$DB_NAME\" OWNER TO \"$DB_USER\";" fi - # Needed on Postgres < 15: owner doesn't get public-schema CREATE by - # default there, and migrations fail with "permission denied for schema public". + # Lets the role create tables in the public schema. sudo -u postgres psql -v ON_ERROR_STOP=1 -d "$DB_NAME" -c "GRANT ALL ON SCHEMA public TO \"$DB_USER\";" fi diff --git a/scripts/setup-rabbitmq.sh b/scripts/setup-rabbitmq.sh index ea56b68..6f6c55b 100755 --- a/scripts/setup-rabbitmq.sh +++ b/scripts/setup-rabbitmq.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # SPDX-License-Identifier: GPL-3.0-only -# Re-runnable: an existing install or vhost/user is left as-is. +# Re-runnable: keeps an existing server, vhost and user, and sets the user's +# password, tags and permissions to this run's. set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -82,7 +83,7 @@ parse_args() { main() { parse_args "$@" require_root - [ -f "$INSTALL_DIR/.env" ] || error "$INSTALL_DIR/.env not found, run install.sh first" + [ -f "$INSTALL_DIR/.env" ] || error "$INSTALL_DIR/.env not found. Run install.sh first." validate_identifier "--broker-vhost" "$BROKER_VHOST" validate_identifier "--broker-user" "$BROKER_USER" if [ "$BROKER_EXISTING" != "1" ] && { [ "$BROKER_HOST" != "127.0.0.1" ] || [ "$BROKER_PORT" != "5672" ]; }; then @@ -93,10 +94,8 @@ main() { [ -n "$BROKER_PASSWORD" ] || error "--broker-password is required with --existing" validate_secret "--broker-password" "$BROKER_PASSWORD" - # rabbitmqctl only works locally, so this checks the management HTTP API - # instead. A user without the "management" tag gets 401 on every endpoint - # here, indistinguishable from a wrong password by status code alone. - # Hence the body sniff below. + # Checks the credentials through the management HTTP API. A 401 whose body + # says "Not management user" means the user lacks the management tag. log "Checking connection to existing RabbitMQ server at $BROKER_HOST:$BROKER_MGMT_PORT" tmp_body=$(mktemp) http_code=$(curl -s -o "$tmp_body" -w '%{http_code}' -u "$BROKER_USER:$BROKER_PASSWORD" \ diff --git a/sdk/README.md b/sdk/README.md index 3d4cbdb..145a024 100644 --- a/sdk/README.md +++ b/sdk/README.md @@ -63,7 +63,7 @@ Raise these so the Publisher can act on the failure. Any other exception is logg |---|---| | `InvalidParamsError` | The request is missing something the adapter needs | | `AuthenticationError` | The user's code, password or authorization was rejected | -| `TokenInvalidError` | The stored token no longer works; the user must link again | +| `TokenInvalidError` | The stored token no longer works and the user must link the account again | | `RateLimitedError` | The platform is throttling, optionally with `retry_after` seconds | | `UpstreamError` | The platform failed or couldn't be reached | @@ -130,11 +130,11 @@ venv/bin/relaysms-adapter revoke | Command | Does | |---|---| -| `link` | OAuth2: opens the authorization URL and catches the redirect on a local `http://localhost` redirect URL, or asks you to paste it. PNBA: asks for the phone number, code and two-step password. | -| `send` | Sends from the linked account, asking for what the category needs. `--to`, `--subject`, `--body` and `--attach` skip the questions; `--offline` sends without an account. | +| `link` | OAuth2: opens the authorization URL, then catches the redirect when the redirect URL is `http://localhost`, or asks you to paste it. PNBA: asks for the phone number, code and two-step password. | +| `send` | Sends from the linked account, asking for what the category needs. `--to`, `--subject`, `--body` and `--attach` skip the questions. `--offline` sends without an account. | | `revoke` | Unlinks the account. | | `setup` | Runs `setup` and saves `credentials.json`. `--input key=value`, or `key=@path` for a file, skips the questions. | -| `config show`, `config set` | Lists the settings, or changes them; `set` asks for a bare `KEY`'s value and takes `--unset KEY`. | +| `config show`, `config set` | Lists or changes the settings. `set` asks for a bare `KEY`'s value and takes `--unset KEY`. | | `call [json]` | Calls any method with raw params. | ## Change the SDK @@ -142,7 +142,7 @@ venv/bin/relaysms-adapter revoke Adapters pin SDK releases, so the Publisher talks to older ones. - Keep requests, results, errors and `adapter.toml` backward compatible. -- Bump `version` in `pyproject.toml` with every change to `src/`; CI checks. +- Bump `version` in `pyproject.toml` with every change to `src/`. CI fails without it. - Release by pushing `sdk-v` plus the version, such as `sdk-v1.1.0`. ## Release an adapter @@ -165,7 +165,7 @@ permissions: jobs: adapter: - uses: smswithoutborders/RelaySMS-Publisher/.github/workflows/adapter.yml@sdk-v1.0.0 + uses: smswithoutborders/RelaySMS-Publisher/.github/workflows/adapter.yml@sdk-v1.1.0 ``` -Set `version` in `pyproject.toml` and push the matching tag, such as `v1.2.0`; the release fails if they differ. The Publisher installs and updates adapters by tag. +Set `version` in `pyproject.toml` and push the matching tag, such as `v1.2.0`. The release fails if they differ. The Publisher installs and updates adapters by tag. diff --git a/sdk/examples/echo-adapter/src/echo_adapter/adapter.py b/sdk/examples/echo-adapter/src/echo_adapter/adapter.py index c10ecb8..814bec1 100644 --- a/sdk/examples/echo-adapter/src/echo_adapter/adapter.py +++ b/sdk/examples/echo-adapter/src/echo_adapter/adapter.py @@ -52,7 +52,7 @@ def send_message(self, request: SendRequest) -> SendResult: @override def revoke(self, request: RevokeRequest) -> None: - logger.info("Unlinked %s", request.account.identifier) + logger.info("Unlinked the account.") @override @classmethod diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index 9320c64..a778618 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "relaysms-adapter-sdk" -version = "1.1.3" +version = "1.1.4" description = "Build RelaySMS Publisher platform adapters" readme = "README.md" license = "GPL-3.0-only" diff --git a/sdk/src/relaysms_adapter_sdk/adapter.py b/sdk/src/relaysms_adapter_sdk/adapter.py index 3095be3..7b33a37 100644 --- a/sdk/src/relaysms_adapter_sdk/adapter.py +++ b/sdk/src/relaysms_adapter_sdk/adapter.py @@ -47,12 +47,9 @@ def revoke(self, request: RevokeRequest) -> MaybeAwaitable[None]: @classmethod def setup(cls, request: SetupRequest) -> MaybeAwaitable[SetupResult]: - """Turn the [setup] inputs into settings, e.g. by registering an app. - - A classmethod, since an adapter's __init__ may read the settings setup creates. - """ + """Turn the [setup] inputs into settings, e.g. by registering an app.""" raise AdapterError( - f"{cls.__name__} has no setup; set its config instead.", + f"{cls.__name__} has no setup. Set its config instead.", code=METHOD_NOT_FOUND, ) diff --git a/sdk/src/relaysms_adapter_sdk/console.py b/sdk/src/relaysms_adapter_sdk/console.py index 88ba0d9..c7c85ba 100644 --- a/sdk/src/relaysms_adapter_sdk/console.py +++ b/sdk/src/relaysms_adapter_sdk/console.py @@ -163,14 +163,13 @@ def revoke(ws: Workspace, _args: argparse.Namespace) -> None: def setup(ws: Workspace, args: argparse.Namespace) -> None: if not ws.manifest.setup: - raise ConsoleError(f"This adapter has no setup; write its {CREDENTIALS_FILE}.") + raise ConsoleError(f"This adapter has no setup. Write its {CREDENTIALS_FILE}.") inputs = manifest.parse_inputs( ws.manifest.setup, args.input, _ask, ask_required=True ) inputs = manifest.check_values(ws.manifest.setup, inputs, require_all=True) result = call(ws, "setup", SetupRequest(inputs)) - # The settings replace the file, so every required one must be there. settings = manifest.check_values( ws.manifest.config, result["settings"], require_all=True ) @@ -250,7 +249,7 @@ def main(argv: list[str] | None = None) -> int: action="append", default=[], metavar="KEY=VALUE", - help="repeatable; KEY=@PATH for a file. Asks for required inputs not given.", + help="repeatable, KEY=@PATH for a file. Asks for required inputs not given.", ) setup_parser.set_defaults(handler=setup) @@ -260,7 +259,7 @@ def main(argv: list[str] | None = None) -> int: handler=config_show ) set_parser = config_commands.add_parser( - "set", help="change settings; a bare KEY asks for its value" + "set", help="change settings (a bare KEY asks for its value)" ) set_parser.add_argument("values", nargs="*", metavar="KEY=VALUE|KEY") set_parser.add_argument("--unset", action="append", default=[], metavar="KEY") @@ -350,7 +349,7 @@ def _link_pnba(ws: Workspace, args: argparse.Namespace) -> dict[str, Any]: def _wait_for_redirect(url: str) -> dict[str, str] | None: - """Catch the redirect on a local server; None when the URL isn't local.""" + """Catch the redirect's query, or return None if not a local URL or none came.""" target = urlparse(url) if target.scheme != "http" or target.hostname not in {"localhost", "127.0.0.1"}: return None @@ -410,7 +409,7 @@ def _load_account(ws: Workspace) -> dict[str, Any]: try: return json.loads(ws.account_file.read_text(encoding="utf-8")) except FileNotFoundError: - raise ConsoleError("No linked account; run relaysms-adapter link.") from None + raise ConsoleError("No linked account. Run relaysms-adapter link.") from None def _save_account(ws: Workspace, account: dict[str, Any]) -> None: diff --git a/sdk/src/relaysms_adapter_sdk/log.py b/sdk/src/relaysms_adapter_sdk/log.py index 77d93a6..c54821f 100644 --- a/sdk/src/relaysms_adapter_sdk/log.py +++ b/sdk/src/relaysms_adapter_sdk/log.py @@ -21,7 +21,7 @@ def format(self, record: logging.LogRecord) -> str: def parse(line: str) -> dict[str, str] | None: - """Read back a line JSONFormatter wrote; None for anything else.""" + """Read back a line JSONFormatter wrote, or return None for any other line.""" try: entry = json.loads(line) except json.JSONDecodeError: diff --git a/sdk/src/relaysms_adapter_sdk/manifest.py b/sdk/src/relaysms_adapter_sdk/manifest.py index 6f30401..c780828 100644 --- a/sdk/src/relaysms_adapter_sdk/manifest.py +++ b/sdk/src/relaysms_adapter_sdk/manifest.py @@ -132,8 +132,7 @@ def check_values( Raises: ConfigError: A key isn't a field, is given twice in different case, has a - value of the wrong type, or is required and missing. The message names - the key, never the value, which may be a secret. + value of the wrong type, or is required and missing. """ fields = tuple(fields) try: @@ -177,14 +176,14 @@ def parse_inputs( ) -> dict[str, Any]: """Turn KEY=VALUE command-line inputs into values, with lowercase keys. - A bare KEY has ask return its value as text; for a file, the file's path. + A bare KEY's value comes from ask: its text, or for a file, the file's path. With ask_required, ask is also called for each required field not given. - An unknown KEY=VALUE keeps its text, so check_values reports it with others. + An unknown KEY=VALUE is returned as text for check_values to report. Every input is checked before anything is asked. Raises: - ConfigError: A key is bare without ask or unknown, is given twice, or its - value can't be converted (see from_text). + ConfigError: A bare key has no ask or isn't a field, a key is given twice, + or a value can't be converted (see from_text). """ by_name = {field.name: field for field in fields} pairs, asked = [], [] @@ -216,7 +215,7 @@ def parse_inputs( def merge_values(current: dict[str, Any], changes: dict[str, Any]) -> dict[str, Any]: - """Apply checked changes to settings; a None value removes the setting.""" + """Apply checked changes to settings, removing those set to None.""" merged = current | changes return {key: value for key, value in merged.items() if value is not None} @@ -244,7 +243,7 @@ def from_text(field: Field, text: str) -> Any: return [item.strip() for item in text.split(",") if item.strip()] case FieldType.FILE: if not text.startswith("@"): - raise ConfigError(f"{field.name} is a file; give it as @path.") + raise ConfigError(f"{field.name} is a file: give it as @path.") return _read_file(field.name, Path(text[1:]).expanduser()) return text @@ -252,7 +251,6 @@ def from_text(field: Field, text: str) -> Any: def _read_file(name: str, path: Path) -> str: try: with path.open("rb") as file: - # One byte over the limit is enough to know it's too big. data = file.read(MAX_FILE_SIZE + 1) text = data.decode("utf-8") except (OSError, UnicodeDecodeError) as e: @@ -277,7 +275,7 @@ def _fields(data: dict[str, Any], section: str, *, files: bool) -> tuple[Field, raise ManifestError(f"{label} must be a table.") kind = _enum(FieldType, table, "type", FieldType.STRING, label) if kind is FieldType.FILE and not files: - raise ManifestError(f"{label}.type can't be file; only setup inputs can.") + raise ManifestError(f"{label}.type can't be file. Only setup inputs can.") fields.append( Field( name=name, diff --git a/sdk/src/relaysms_adapter_sdk/paths.py b/sdk/src/relaysms_adapter_sdk/paths.py index b085c63..8763ff6 100644 --- a/sdk/src/relaysms_adapter_sdk/paths.py +++ b/sdk/src/relaysms_adapter_sdk/paths.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-3.0-only """The directories the Publisher keeps for an adapter, outside its code. -Updates replace the code, so secrets go in the config directory and anything -the adapter writes goes in the state directory. +Secrets go in the config directory and anything the adapter writes goes in the +state directory. """ import json @@ -40,9 +40,9 @@ def read_config( ) -> dict[str, Any]: """Read a JSON object from the config directory, with every key in lowercase. - So `SMTP_HOST` and `smtp_host` both read as `smtp_host`, in nested objects too. - Pass directory to read another adapter's file: the Publisher does, because - RELAYSMS_ADAPTER_CONFIG_DIR is only set inside an adapter process. + `SMTP_HOST` and `smtp_host` both read as `smtp_host`, in nested objects too. + Pass directory to read a file outside an adapter process, where + RELAYSMS_ADAPTER_CONFIG_DIR isn't set. Raises: FileNotFoundError: The file doesn't exist. @@ -89,7 +89,7 @@ def lowercase_keys(pairs: Iterable[tuple[str, Any]]) -> dict[str, Any]: for key, value in pairs: key = key.lower() if key in lowered: - raise ValueError(f"{key!r} is set twice; keys ignore case.") + raise ValueError(f"{key!r} is set twice (keys ignore case).") lowered[key] = value return lowered diff --git a/sdk/src/relaysms_adapter_sdk/runner.py b/sdk/src/relaysms_adapter_sdk/runner.py index ade278d..89df544 100644 --- a/sdk/src/relaysms_adapter_sdk/runner.py +++ b/sdk/src/relaysms_adapter_sdk/runner.py @@ -23,7 +23,7 @@ def run( stdout: TextIO = sys.stdout, ): """Answer each request line until standard input closes.""" - # One loop for the process, so async clients can outlive a request. + # Every async method runs on this one event loop. with asyncio.Runner() as runner: for line in stdin: if line.strip(): @@ -37,8 +37,7 @@ def handle( """Answer one JSON-RPC request line. Given the class, it creates an adapter for each request, except setup, which - runs on the class because an adapter's __init__ may read the settings setup - creates. + runs on the class. """ try: request = wire.parse_request(line) diff --git a/sdk/src/relaysms_adapter_sdk/wire.py b/sdk/src/relaysms_adapter_sdk/wire.py index da2115d..8fa294b 100644 --- a/sdk/src/relaysms_adapter_sdk/wire.py +++ b/sdk/src/relaysms_adapter_sdk/wire.py @@ -44,7 +44,7 @@ def parse_request(line: str) -> Request: """Parse one request line. Raises: - AdapterError: With PARSE_ERROR or INVALID_REQUEST, as JSON-RPC requires. + AdapterError: With PARSE_ERROR or INVALID_REQUEST. """ message = _loads(line) request_id = message.get("id") diff --git a/sdk/tests/test_log_and_paths.py b/sdk/tests/test_log_and_paths.py index 081edca..f77645e 100644 --- a/sdk/tests/test_log_and_paths.py +++ b/sdk/tests/test_log_and_paths.py @@ -114,7 +114,7 @@ def test_write_config_replaces_the_file_for_its_owner_only( assert (tmp_path / "config").stat().st_mode & 0o777 == 0o700 assert [p.name for p in (tmp_path / "config").iterdir()] == ["credentials.json"] - def test_unset(self, monkeypatch): + def test_config_dir_needs_its_env_var(self, monkeypatch): monkeypatch.delenv(CONFIG_DIR_ENV, raising=False) with pytest.raises(RuntimeError, match=CONFIG_DIR_ENV): config_dir() diff --git a/sdk/tests/test_manifest.py b/sdk/tests/test_manifest.py index 3cbe067..90062d4 100644 --- a/sdk/tests/test_manifest.py +++ b/sdk/tests/test_manifest.py @@ -173,7 +173,7 @@ def test_from_text_refuses_bad_files(tmp_path, text, error): manifest.from_text(CLIENT_FILE, text.format(tmp=tmp_path)) -def test_parse_inputs(): +def test_parse_inputs_converts_known_keys_and_keeps_unknown_ones(): fields = (Field("port", FieldType.INTEGER),) assert manifest.parse_inputs(fields, ["PORT=465", "colour=red"]) == { "port": 465, @@ -197,7 +197,7 @@ def test_parse_inputs_refuses(texts, error): (Field("tls", FieldType.BOOLEAN), "yes"), ], ) -def test_from_text_rejects_other_types(field, text): +def test_from_text_rejects_text_of_the_wrong_type(field, text): with pytest.raises(ConfigError, match=field.name): manifest.from_text(field, text) diff --git a/sdk/tests/test_wire.py b/sdk/tests/test_wire.py index fd70b6a..aa5896d 100644 --- a/sdk/tests/test_wire.py +++ b/sdk/tests/test_wire.py @@ -73,12 +73,6 @@ def test_errors_carry_a_refreshed_token(self): wire.parse_response(line) assert (e.value.token, e.value.retry_after) == ({"t": 2}, 30) - def test_rate_limit_keeps_retry_after(self): - line = wire.encode_error(1, RateLimitedError("slow down", retry_after=30)) - with pytest.raises(RateLimitedError) as e: - wire.parse_response(line) - assert e.value.retry_after == 30 - def test_unknown_code(self): line = wire.encode_error(1, AdapterError("odd", code=-32099)) with pytest.raises(AdapterError) as e: @@ -153,7 +147,7 @@ def test_invalid(self, data, error): with pytest.raises(InvalidParamsError, match=error): wire.from_json(SendRequest, data) - def test_bool_is_not_str_or_int(self): + def test_bool_is_not_a_str(self): with pytest.raises(InvalidParamsError, match="must be str"): wire.from_json(Account, {"identifier": True}) diff --git a/sync-env.sh b/sync-env.sh index 9fb2928..3a9a4f0 100755 --- a/sync-env.sh +++ b/sync-env.sh @@ -203,7 +203,7 @@ main() { [ "$PRUNE" -eq 0 ] || prune_env_lines if [ "$added" -eq 0 ] && [ "$removed" -eq 0 ]; then - log "Nothing to change; $ENV_FILE matches the variables in $TEMPLATE_FILE." + log "Nothing to change: $ENV_FILE matches the variables in $TEMPLATE_FILE." return fi local name color=never changes="$added variable(s) added" @@ -225,7 +225,7 @@ main() { case "$answer" in y | Y | yes | YES) ;; *) - log "Aborted; $ENV_FILE left unchanged." + log "Aborted: $ENV_FILE left unchanged." return ;; esac @@ -234,7 +234,7 @@ main() { printf '%s\n' "${env_lines[@]}" >"$ENV_FILE" log "Updated $ENV_FILE ($changes)." [ "$current" = /dev/null ] || - log "Previous file backed up to $ENV_FILE.bak; delete it once you've confirmed the sync looks right." + log "Previous file backed up to $ENV_FILE.bak. Delete it once you've confirmed the sync looks right." } main "$@" diff --git a/template.env b/template.env index 6071082..187c0ff 100644 --- a/template.env +++ b/template.env @@ -15,7 +15,8 @@ GRPC_TLS_ENABLED=false GRPC_TLS_CERT_FILE= GRPC_TLS_KEY_FILE= -# Requests older than this window or with replayed nonces will be rejected. +# Seconds a gRPC request's timestamp may be from the server's clock, and how long +# nonces are remembered to refuse replays. NONCE_TTL_SECONDS=600 # Database @@ -26,7 +27,7 @@ DATABASE_ENCRYPTION_ENABLED=true # 64-char hex (32 bytes). Generate: openssl rand -hex 32 DATABASE_ENCRYPTION_KEY= -# Database Field Encryption (AES-256-GCM for sensitive columns) +# Encrypt linked accounts' platform tokens DATABASE_FIELD_ENCRYPTION_ENABLED=true # 64-char hex (32 bytes). Generate: openssl rand -hex 32 DATABASE_FIELD_ENCRYPTION_KEY= @@ -87,7 +88,7 @@ AUDIT_RETENTION_DAYS=365 # Offline Payload Publishing # Comma-separated allowlist of ingestion protocols allowed to publish offline -# payloads, e.g. smtp,https. Empty allows all protocols (no restriction). +# payloads, e.g. smtp,https. Empty allows every protocol. OFFLINE_PUBLISH_ALLOWED_PROTOCOLS=smtp # Shared secret that offline payloads over https must carry in their "tag" field. @@ -96,7 +97,7 @@ OFFLINE_PUBLISH_ALLOWED_PROTOCOLS=smtp OFFLINE_PUBLISH_SHARED_SECRET= # SMTP Ingestion Transport -# Mail body must be JSON: {"address": "", "text": ""} +# Mail body format: docs/configuration/smtp-transport.md#message-format SMTP_TRANSPORT_ENABLED=false SMTP_IMAP_SERVER= SMTP_IMAP_PORT=993 @@ -116,7 +117,7 @@ SMTP_ALLOWED_SENDERS= SMTP_TRUSTED_AUTHSERV_ID= SMTP_REQUIRE_DKIM=true SMTP_REQUIRE_SPF=true -# Additionally re-verify DKIM via DNS (defense-in-depth) +# Also re-verify the DKIM signature against DNS SMTP_VERIFY_DKIM_INDEPENDENTLY=false # Platform Adapters diff --git a/tests/api/grpc/test_v3.py b/tests/api/grpc/test_v3.py index 426c31f..cbe3e0d 100644 --- a/tests/api/grpc/test_v3.py +++ b/tests/api/grpc/test_v3.py @@ -60,7 +60,7 @@ def server_keys(monkeypatch): keys.initialize_server_identity_keys(s) add_adapter("gmail", proto_id=0, cat_id=1) add_adapter("telegram", proto_id=1, cat_id=2) - # The client helpers fetch server public keys over REST; read them from the db. + # Read server public keys from the db instead of over REST. monkeypatch.setattr( client_helpers, "fetch_server_identity_public_key", @@ -142,10 +142,7 @@ def decrypt_token(response, keypairs): def encrypt_token(response, keypairs, token, key_id=1): - """Encrypt a token with a key pair the server still holds, as revoke does. - - Exchange puts the token in a slot from 16-255, so lower slots stay held. - """ + """Encrypt a token as revoke does, with a slot (0-15) the server still holds.""" ciphertext = rrs.v1_token_encrypt_client( ec_kid=keypairs[key_id].private_bytes_raw(), ss_kid_pk=client_helpers.fetch_server_identity_public_key("", key_id), @@ -308,7 +305,9 @@ def test_other_services_skip_request_auth(): assert result == "served" -def test_get_oauth2_authorization_url(stub, adapter): +def test_authorization_url_comes_from_the_adapter_with_a_generated_verifier( + stub, adapter +): adapter.results["create_authorization_url"] = { "url": "https://auth.example.org/authorize", "state": "state-1", @@ -400,7 +399,7 @@ def test_exchange_oauth2_code_needs_client_key_ids_0_to_255(stub, adapter, key_i assert "each key_id 0-255 exactly once" in error.details() -def test_revoke_oauth2_token(stub, adapter): +def test_revoke_oauth2_token_deletes_it_and_revokes_upstream(stub, adapter): exchanged, keypairs = exchange_oauth2(stub, adapter) payload = encrypt_token(exchanged, keypairs, decrypt_token(exchanged, keypairs)) request = publisher_pb2.RevokeOAuth2TokenRequest( @@ -479,7 +478,7 @@ def test_revoke_deletes_the_token_even_if_the_platform_fails(stub, adapter): assert stored_token(exchanged.token_id) is None -def test_get_pnba_code(stub, adapter): +def test_get_pnba_code_returns_the_adapter_message_and_expiry(stub, adapter): adapter.results["send_code"] = { "message": "Code sent", "expires_at": "2026-10-01T12:00:00+00:00", @@ -583,7 +582,7 @@ def test_exchange_pnba_code_stores_session(stub, adapter): assert hashlib.sha256(raw_token).digest() == token.hash -def test_revoke_pnba_token(stub, adapter): +def test_revoke_pnba_token_deletes_it_and_revokes_upstream(stub, adapter): exchanged, keypairs = exchange_pnba(stub, adapter) payload = encrypt_token(exchanged, keypairs, decrypt_token(exchanged, keypairs)) request = publisher_pb2.RevokePNBATokenRequest( diff --git a/tests/api/rest/test_adapter_config.py b/tests/api/rest/test_adapter_config.py index df4e34f..f22cd35 100644 --- a/tests/api/rest/test_adapter_config.py +++ b/tests/api/rest/test_adapter_config.py @@ -106,7 +106,6 @@ def test_setup_passes_file_inputs_as_text(client, admin, credentials, adapter_ca "settings": {"smtp_host": "h", "smtp_password": "p"} } - # A path is just text here: the server never reads files a client names. response = client.post( SETUP, json={"inputs": {"client_file": "@/etc/hostname"}}, diff --git a/tests/api/rest/test_audit.py b/tests/api/rest/test_audit.py index 1a517b3..fd21da0 100644 --- a/tests/api/rest/test_audit.py +++ b/tests/api/rest/test_audit.py @@ -100,7 +100,7 @@ def test_denied_change_is_recorded_and_rolled_back(client, admin): assert "gc:read" in event["details"]["reason"] -def test_logins_and_logouts(client, admin, password): +def test_logins_and_logouts_are_recorded(client, admin, password): assert login(client, "wrong", username=USERNAME).status_code == 401 assert login(client, "wrong", username="nobody").status_code == 401 assert ( @@ -200,7 +200,7 @@ def test_rejects_bad_query(client, admin, params, status): assert response.status_code == status -def test_delete_older_than(): +def test_delete_older_than_removes_old_events(): create_credential("analyst", ["gc:read"]) with db.get_session() as session: old = session.query(AuditEvent).filter_by(target_label="analyst").one() diff --git a/tests/api/rest/test_auth.py b/tests/api/rest/test_auth.py index 0660ddf..47ac102 100644 --- a/tests/api/rest/test_auth.py +++ b/tests/api/rest/test_auth.py @@ -74,7 +74,9 @@ def test_login_rejects_bad_credentials_generically( ({"Origin": "null"}, 403), ], ) -def test_login_origin_check(client, password, set_config, headers, status): +def test_login_accepts_only_its_own_or_configured_origins( + client, password, set_config, headers, status +): set_config(auth, "auth_config", web_origins=[WEB_ORIGIN]) response = client.post( @@ -99,7 +101,6 @@ def test_me_unauthenticated_is_401_without_basic_prompt(client): response = client.get("/v1/auth/me") assert response.status_code == 401 - # A Basic challenge would make browsers show a login dialog. assert "www-authenticate" not in response.headers @@ -123,7 +124,7 @@ def session_write(context=Depends(authenticate_request)): set_config(auth, "auth_config", web_origins=[WEB_ORIGIN]) login(client, password) - # A sibling subdomain is same-site, so SameSite alone would let it through. + # A same-site sibling of the web origin. sibling = {"Origin": "https://other.example.net"} assert client.post("/v1/session-write", headers=sibling).status_code == 403 ok = client.post("/v1/session-write", headers={"Origin": WEB_ORIGIN}) diff --git a/tests/api/rest/test_creds.py b/tests/api/rest/test_creds.py index 396b6e3..4e01bd1 100644 --- a/tests/api/rest/test_creds.py +++ b/tests/api/rest/test_creds.py @@ -10,7 +10,7 @@ @pytest.fixture def analyst_session(app): - """A separate client, since a session cookie takes precedence over Basic auth.""" + """A second client, logged in as analyst with a session cookie.""" session_client = TestClient(app, base_url="https://testserver") password = create_credential("analyst", ["stats:publications:read"]) login(session_client, password, username="analyst") @@ -53,7 +53,7 @@ def test_endpoints_require_auth_and_scope(client, analyst, method, url): assert response.json()["error"].startswith("Missing scope: creds:") -def test_list_and_get(client, admin, analyst): +def test_list_and_get_show_scopes_without_etags(client, admin, analyst): listed = client.get("/v1/creds", headers=admin) assert listed.status_code == 200 @@ -76,7 +76,7 @@ def test_etags_are_opaque(client, admin, analyst): # Both are at version 1, yet their etags differ and don't show it. assert etags[USERNAME] != etags["analyst"] - assert all(len(etag) == 34 and etag not in ('"1"', "1") for etag in etags.values()) + assert all(len(etag) == 34 for etag in etags.values()) def test_create_returns_a_working_password_once(client, admin): @@ -133,12 +133,12 @@ def test_update_requires_a_current_if_match(client, admin, analyst, caplog): missing = client.patch(url, json=body, headers=admin) stale = client.patch(url, json=body, headers={**admin, "If-Match": '"999"'}) - # Weak, as nginx gzip sends it back. + # The weak form nginx gzip sends back. ok = client.patch(url, json=body, headers={**admin, "If-Match": f"W/{etag}"}) assert missing.status_code == 428 assert stale.status_code == 412 - # The client gets a plain message; the comparison goes to the log. + # The comparison goes to the log, not the response. assert '"999"' not in stale.json()["error"] assert 'If-Match "999" != ETag' in caplog.text assert ok.status_code == 200, ok.text @@ -180,14 +180,14 @@ def test_reset_password_replaces_the_old_one(client, admin): ) -def test_revoke_sessions(client, admin, analyst_session): +def test_revoke_sessions_logs_out_existing_sessions(client, admin, analyst_session): response = client.post("/v1/creds/analyst/revoke-sessions", headers=admin) assert response.status_code == 204 assert analyst_session.get("/v1/auth/me").status_code == 401 -def test_delete(client, admin, analyst): +def test_delete_needs_if_match_and_removes_the_credential(client, admin, analyst): url = "/v1/creds/analyst" assert client.delete(url, headers=admin).status_code == 428 diff --git a/tests/api/rest/test_platforms.py b/tests/api/rest/test_platforms.py index 364a2d7..293878f 100644 --- a/tests/api/rest/test_platforms.py +++ b/tests/api/rest/test_platforms.py @@ -56,7 +56,6 @@ def test_oauth_callback_is_404_for_unknown_platforms(client): def test_client_metadata_comes_from_the_config_dir(client, platforms_config): config = PlatformsConfig.get().adapters_config_dir / "bluesky-0" config.mkdir(parents=True) - # The adapter accepts uppercase keys, so this route must too. uppercase = {key.upper(): value for key, value in CLIENT_METADATA.items()} (config / "credentials.json").write_text(json.dumps(uppercase)) diff --git a/tests/api/rest/test_stats.py b/tests/api/rest/test_stats.py index 53615e0..9303085 100644 --- a/tests/api/rest/test_stats.py +++ b/tests/api/rest/test_stats.py @@ -237,7 +237,7 @@ def test_filters_and_time_range(authed_client, seeded): ({"since": "2026-09-02T00:00:00Z", "until": "2026-09-01T00:00:00Z"}, 400), ], ) -def test_invalid_params(authed_client, seeded, params, status): +def test_bad_cursor_or_reversed_range_is_400(authed_client, seeded, params, status): assert ( authed_client.get("/v1/stats/publications", params=params).status_code == status ) @@ -325,7 +325,7 @@ def test_summary_interval_buckets_by_week(authed_client): ] -def test_summary_interval_with_multiple_group_by(authed_client): +def test_summary_interval_combines_with_group_by(authed_client): _seed( [ dict( diff --git a/tests/cli/test_gateway_clients.py b/tests/cli/test_gateway_clients.py index 87562c4..85d3a6f 100644 --- a/tests/cli/test_gateway_clients.py +++ b/tests/cli/test_gateway_clients.py @@ -38,7 +38,7 @@ def test_create_update_disable_and_delete(): assert _clients() == [] -def test_errors_exit_non_zero(): +def test_deleting_an_unknown_client_exits_1(): result = run("delete", MSISDN) assert result.exit_code == 1 diff --git a/tests/cli/test_platforms.py b/tests/cli/test_platforms.py index 84b1286..e6aa8e3 100644 --- a/tests/cli/test_platforms.py +++ b/tests/cli/test_platforms.py @@ -5,7 +5,6 @@ import pytest from click.testing import CliRunner -from git import Repo from publisher import db from publisher.cli import platforms as platforms_cli @@ -76,21 +75,16 @@ def _run(*args): return CliRunner().invoke(platforms_cli.cli, list(args)) -def test_add_list_and_import(tmp_path): +def test_add_and_list(tmp_path): url = str(adapter_repo(tmp_path / "src" / "mastodon", "mastodon").working_tree_dir) added = _run("add", url) assert added.exit_code == 0, added.output assert "Adapter 'mastodon' added at v1.0.0" in added.output - source = adapter_repo(tmp_path / "src" / "telegram", "telegram", PNBA) - Repo.clone_from(str(source.working_tree_dir), tmp_path / "adapters" / "tg-id") - imported = _run("import") - assert imported.output.strip() == "Imported 1 adapter(s). telegram" - listed = _run("list") names = [line.split("|")[2].strip() for line in listed.output.splitlines()[2:]] - assert names == ["gmail", "mastodon", "telegram"] + assert names == ["gmail", "mastodon"] @pytest.mark.parametrize( diff --git a/tests/cli/test_seed.py b/tests/cli/test_seed.py index 0ae85dd..0ed2c47 100644 --- a/tests/cli/test_seed.py +++ b/tests/cli/test_seed.py @@ -25,7 +25,11 @@ def test_stats_adds_realistic_rows_in_batches(monkeypatch): with get_session() as db: rows = db.query(PublicationStats).all() assert len(rows) == 50 - assert all((row.status == "failed") == bool(row.failure_reason) for row in rows) + assert all((row.status != "published") == bool(row.failure_reason) for row in rows) + assert all( + (row.status == "rejected") == (row.failure_reason in seed.REJECTION_REASONS) + for row in rows + ) times = [row.created_at for row in rows] assert max(times) - min(times) <= datetime.timedelta(days=3) diff --git a/tests/conftest.py b/tests/conftest.py index 671a1ce..3cf142f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import pytest -# Set before config is imported, so tests ignore the local .env and use fixed values. +# Fixed test settings, with the local .env ignored. Set before config is imported. os.environ.update( LOAD_DOTENV="false", DATABASE_DIALECT="sqlite", diff --git a/tests/dialects/conftest.py b/tests/dialects/conftest.py index b4c79a7..eac94a5 100644 --- a/tests/dialects/conftest.py +++ b/tests/dialects/conftest.py @@ -38,8 +38,7 @@ class Server: POSTGRES_FLAGS = ("fsync=off", "synchronous_commit=off", "full_page_writes=off") MYSQL_FLAGS = ("--innodb-flush-log-at-trx-commit=0", "--innodb-doublewrite=0") -# What Ubuntu 24.04 installs. The data is thrown away, so it lives in memory and -# nothing waits for the disk. +# The versions Ubuntu 24.04 installs, with data on tmpfs and disk syncs off. SERVERS = { "postgres": Server( "docker.io/library/postgres:16", diff --git a/tests/dialects/test_dialects.py b/tests/dialects/test_dialects.py index 6373020..e250842 100644 --- a/tests/dialects/test_dialects.py +++ b/tests/dialects/test_dialects.py @@ -3,26 +3,32 @@ import datetime import hashlib -import json +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace import pytest from alembic import command from alembic.autogenerate import compare_metadata from alembic.migration import MigrationContext -from sqlalchemy import LargeBinary, select, text +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey +from sqlalchemy import LargeBinary, func, select, text from sqlalchemy.dialects import mysql from sqlalchemy.exc import IntegrityError -from publisher import credentials, db, keys +from lib_relaysms_payload_specs.generated import relaysms_spec_payload as rrs +from publisher import credentials, db, keys, publications from publisher.credentials import CredentialConflictError, CredentialExistsError from publisher.db import Base, pagination from publisher.models import audit_event, publication_stats -from publisher.models import gateway_client as gateway_clients from publisher.models import platform_adapter_job as jobs from publisher.models import token as tokens from publisher.models import token_hash as token_hashes from publisher.models.audit_event import AuditAction from publisher.models.credential import ALL_SCOPES +from publisher.models.payload_segment import PayloadSegment +from publisher.models.payload_session import PayloadSession from publisher.models.platform_adapter import PlatformAdapter from publisher.models.publication_stats import PublicationStats, StatsFilters from publisher.models.server_identity_key import get_private_key, get_public_keys @@ -68,33 +74,6 @@ def test_migrations_downgrade_and_upgrade_again(alembic_config): command.upgrade(alembic_config, "head") -def test_migration_imports_the_gateway_client_registry( - alembic_config, tmp_path, monkeypatch -): - registry = tmp_path / "registry.json" - client = { - "msisdn": "+237670000000", - "country": UNICODE, - "operator": "MTN Cameroon", - "operator_code": "62401", - "protocols": ["https", "sms"], - } - incomplete = {"msisdn": "+237690000000", "protocols": ["sms"]} - registry.write_text(json.dumps({c["msisdn"]: c for c in (client, incomplete)})) - monkeypatch.setenv("GATEWAY_CLIENTS_REGISTRY_FILE", str(registry)) - - command.downgrade(alembic_config, "018") - command.upgrade(alembic_config, "head") - - with db.get_session() as session: - assert len(gateway_clients.find(session, include_disabled=True)) == 1 - [imported] = gateway_clients.find(session, country=UNICODE.upper()) - assert (imported.msisdn, imported.protocols) == ( - client["msisdn"], - ["https", "sms"], - ) - - def _same_type(context, inspected_column, metadata_column, inspected, metadata): # MySQL creates LargeBinary(32) as TINYBLOB, the smallest blob that fits. if ( @@ -293,3 +272,117 @@ def test_audit_details_keep_json_and_unicode(): page = audit_event.list_events(session, scopes=ALL_SCOPES, limit=10) assert [event.details for event in page.items] == [details] + + +def _segments() -> list[str]: + payload = rrs.V1Payloads( + contents=b"x" * 1000, k_id=5, len_att=600, t_id=7, sess_id=9 + ) + return [segment.decode() for segment in payload.split(rrs.Transports.SMS)] + + +def _store_segment(session, sms: str): + payload_raw, raw_segment, _ = publications.validate(sms) + return publications._store_segment_and_try_join( + session, sender_id="sender", payload_raw=payload_raw, raw_segment=raw_segment + ) + + +def _in_overlapping_transactions(work_first, work_second): + """Run two functions in overlapping transactions, as two workers would.""" + first_done, second_started = threading.Event(), threading.Event() + + def run_first(): + with db.get_session() as session: + result = work_first(session) + first_done.set() + assert second_started.wait(5), "the second worker never started" + time.sleep(0.2) # Lets the second reach the lock before this commits. + return result + + def run_second(): + first_done.wait(5) + second_started.set() + with db.get_session() as session: + return work_second(session) + + with ThreadPoolExecutor(2) as pool: + return pool.submit(run_first), pool.submit(run_second) + + +def _store_both(first: str, second: str): + futures = _in_overlapping_transactions( + lambda session: _store_segment(session, first), + lambda session: _store_segment(session, second), + ) + return tuple(future.result() for future in futures) + + +def test_segments_arriving_together_share_one_session(): + first, second, *_ = _segments() + + assert _store_both(first, second) == (None, None) + + with db.get_session() as session: + assert session.scalar(select(func.count(PayloadSession.id))) == 1 + assert session.scalar(select(func.count(PayloadSegment.id))) == 2 + + +def test_the_last_segments_arriving_together_are_joined(): + *earlier, second_last, last = _segments() + with db.get_session() as session: + for segment in earlier: + assert _store_segment(session, segment) is None + + first, second = _store_both(second_last, last) + + assert first is None + assert second is not None + with db.get_session() as session: + assert session.scalar(select(func.count(PayloadSession.id))) == 0 + + +SLOT = 20 + + +def _token_with_key_pools() -> tuple[int, tuple[bytes, bytes]]: + """Create a token with key pools, returning its hash id and SLOT's public keys.""" + client_keys = [X25519PrivateKey.generate() for _ in range(256)] + client_public = [k.public_key().public_bytes_raw() for k in client_keys] + with db.get_session() as session: + keys.initialize_server_identity_keys(session) + token = tokens.create(session, "gmail", 0, 0, {"account_id": "a", "token": {}}) + token_hash, _ = token_hashes.create(session, token.id) + server_public = keys.sync_token_pools( + session, + token_hash, + [ + SimpleNamespace(key_id=i, public_key=pk) + for i, pk in enumerate(client_public) + ], + ) + return token_hash.id, (server_public[SLOT], client_public[SLOT]) + + +def test_a_key_slot_can_be_used_once(): + token_hash_id, public_keys = _token_with_key_pools() + + with db.get_session() as session: + assert keys.pop_keys(session, token_hash_id, SLOT)[2:] == public_keys + + with pytest.raises(keys.KeyUnavailableError), db.get_session() as session: + keys.pop_keys(session, token_hash_id, SLOT) + + +def test_one_of_two_workers_using_a_key_slot_at_once_gets_it(caplog): + token_hash_id, public_keys = _token_with_key_pools() + + def pop(session): + return keys.pop_keys(session, token_hash_id, SLOT) + + first, second = _in_overlapping_transactions(pop, pop) + + assert first.result()[2:] == public_keys + with pytest.raises(keys.KeyUnavailableError): + second.result() + assert "Server ephemeral key unavailable" in caplog.text diff --git a/tests/e2e/Containerfile b/tests/e2e/Containerfile index 613f926..afeb3a2 100644 --- a/tests/e2e/Containerfile +++ b/tests/e2e/Containerfile @@ -1,8 +1,7 @@ ARG BASE=docker.io/library/ubuntu:24.04 FROM ${BASE} -# systemd runs as PID 1. install.sh's packages and Rust are baked in so each -# run only spends time on the publisher itself. +# systemd runs as PID 1. The image has install.sh's packages and Rust preinstalled. RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ systemd systemd-sysv dbus sudo ca-certificates curl git openssl make \ diff --git a/tests/e2e/test_install.py b/tests/e2e/test_install.py index c3090e8..8f0c25e 100644 --- a/tests/e2e/test_install.py +++ b/tests/e2e/test_install.py @@ -24,7 +24,7 @@ } D = "/opt/relaysms/relaysms-publisher" UNITS = "/etc/systemd/system/relaysms-publisher-*.service" -# smtp exits cleanly while SMTP_TRANSPORT_ENABLED is off, so it isn't checked. +# Every service but smtp, which exits while SMTP_TRANSPORT_ENABLED is off. SERVICES = " ".join( f"relaysms-publisher-{name}.service" for name in ("rest", "grpc", "worker", "beat") ) @@ -36,15 +36,6 @@ "data/platforms/state", ] DATA_RW_PATHS = "ReadWritePaths=" + " ".join(f"{D}/{path}" for path in DATA_DIRS) -# Env var, then the path before and after scripts/migrate-runtime-data.sh. -LEGACY = { - "PLATFORMS_ADAPTERS_DIR": ("platforms/adapters", "data/platforms/adapters"), - "PLATFORMS_ADAPTERS_VENV_DIR": ("platforms/adapters_venv", "data/platforms/venvs"), - "PLATFORMS_ADAPTERS_STATE_DIR": ( - "platforms/adapters_assets", - "data/platforms/state", - ), -} def run(*args: str) -> str: @@ -132,9 +123,8 @@ def container(request, source): image, ) # fmt: skip try: - # Just after the container starts, systemctl fails until systemd is ready, - # and boot later empties /tmp, where the install clones. So wait for boot to - # finish before any test runs. + # Wait for boot to finish. Before then systemctl fails, and boot empties + # /tmp, where the install clones. booted = "systemctl is-system-running --wait | grep -qxE 'running|degraded'" wait_until(name, booted) yield name @@ -170,12 +160,7 @@ def test_cli_credential_signs_in_to_api(container): ok(container, f"curl -fsS -u 'ops:{password[1]}' 127.0.0.1:16000/v1/auth/me") -def test_update_moves_legacy_layout_and_refreshes_units(container): - moves = "\n".join( - f"mv {new} {old} && sed -i 's#^{var}=.*#{var}={old}#' .env" - for var, (old, new) in LEGACY.items() - ) - legacy_paths = " ".join(f"{D}/{old}" for old, _ in LEGACY.values()) +def test_update_refreshes_units_after_a_rewritten_branch(container): ok( container, f"""set -e @@ -184,20 +169,12 @@ def test_update_moves_legacy_layout_and_refreshes_units(container): # Diverges from upstream, as a force-pushed branch leaves the checkout. git -c user.name=e2e -c user.email=e2e@localhost -c commit.gpgsign=false \ commit -q --amend -m rewritten - mkdir platforms && chown --reference=. platforms - {moves} - sed -i 's#^PLATFORMS_ADAPTERS_STATE_DIR=#PLATFORMS_ADAPTERS_ASSETS_DIR=#' .env - mkdir platforms/adapters/demo - touch platforms/adapters/demo/main.py - sed -i 's#^ReadWritePaths=.*#ReadWritePaths={D}/data {legacy_paths}#' {UNITS} + sed -i 's#^ReadWritePaths=.*#ReadWritePaths={D}/data {D}/stale#' {UNITS} systemctl daemon-reload""", ) ok(container, f"{D}/manage.sh update", timeout=1800) - ok(container, f"test -f {D}/data/platforms/adapters/demo/main.py") - for var, (_, new) in LEGACY.items(): - assert ok(container, f"grep ^{var}= {D}/.env") == f"{var}={new}\n" assert read_write_paths(container) == {DATA_RW_PATHS} assert_serving(container) diff --git a/tests/fixtures.py b/tests/fixtures.py index 169bb66..35965e8 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -52,7 +52,7 @@ def app(): @pytest.fixture def client(app): - # https, or the Secure cookie isn't sent back. + # The Secure session cookie is only sent back over https. return TestClient(app, base_url="https://testserver") @@ -78,7 +78,7 @@ def writer(): @pytest.fixture def platforms_config(tmp_path, monkeypatch): - """Points the adapter directories at tmp_path; call it to change other fields.""" + """Points the adapter directories at tmp_path. Call it to change other fields.""" config = dataclasses.replace( PlatformsConfig.get(), adapters_dir=tmp_path / "adapters", diff --git a/tests/gateway_clients/test_mcc_mnc.py b/tests/gateway_clients/test_mcc_mnc.py index aee5579..c28a2c9 100644 --- a/tests/gateway_clients/test_mcc_mnc.py +++ b/tests/gateway_clients/test_mcc_mnc.py @@ -18,7 +18,7 @@ def _networks(carrier, iso, country_code): ("Glo Mobile Nigeria", "ng", 234, [("62150", "Glo Mobile")]), # Words split differently, or one of several "/" names. ("Beeline", "uz", 998, [("43404", "Bee Line/Unitel")]), - # NANP regions share calling code 1, so the region decides. + # NANP regions share calling code 1 and are told apart by region. ("Viva", "do", 1, [("37004", "Viva")]), # Guernsey has no rows of its own and uses the UK's. ("Sure", "gg", 44, [("23455", "Sure Guernsey")]), @@ -49,6 +49,6 @@ def test_find_matches_searches_the_snapshot_ignoring_case_and_accents(): [match] = mcc_mnc.find_matches(country_code="237", network="mtn", iso="cm") assert (match["mcc"], match["mnc"]) == ("624", "01") - assert mcc_mnc.find_matches(network="TELEFÓNICA") == mcc_mnc.find_matches( - network="telefonica" - ) + telefonica = mcc_mnc.find_matches(network="telefonica") + assert telefonica + assert mcc_mnc.find_matches(network="TELEFÓNICA") == telefonica diff --git a/tests/helpers.py b/tests/helpers.py index 4c38b5c..0f12916 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-3.0-only -"""Helpers for tests that create credentials and log in.""" +"""Helpers that create credentials, adapters, linked accounts and adapter repos.""" import base64 from collections.abc import Iterable @@ -26,7 +26,7 @@ def create_credential(username: str, scopes: Iterable[str] = ALL_SCOPES) -> str: def add_adapter(name: str, proto_id: int, cat_id: int = 0) -> PlatformAdapter: - """Register an adapter row; its id is "-".""" + """Register an adapter row with id "-".""" with db.get_session() as session: adapter = PlatformAdapter( id=f"{name}-{proto_id}", @@ -52,7 +52,7 @@ def add_adapter_with_manifest( def link_account(platform: str, proto_id: int) -> int: - """Store a token as linking an account would; return its row id.""" + """Store a token as linking an account would and return its row id.""" with db.get_session() as session: token = create_token( session, @@ -112,7 +112,7 @@ def adapter_repo( def commit_manifest( repo: Repo, name: str, proto_id: int, tag: str | None = None ) -> str: - """Write and commit adapter.toml, tagged if tag; return the commit's sha.""" + """Write and commit adapter.toml, tag it if tag is set, and return its sha.""" path = repo.working_tree_dir protocol = PROTOCOL_NAMES[proto_id] with open(f"{path}/adapter.toml", "w") as f: diff --git a/tests/platforms/test_manager.py b/tests/platforms/test_manager.py index 91763d9..4ddf390 100644 --- a/tests/platforms/test_manager.py +++ b/tests/platforms/test_manager.py @@ -3,7 +3,6 @@ import subprocess import pytest -from git import Repo from sqlalchemy import select from publisher import db @@ -283,60 +282,6 @@ def fail(path, venv, log): assert _rows() == [] -@pytest.mark.usefixtures("test_db") -def test_import_registers_unregistered_clones_only(tmp_path): - adapters_dir = tmp_path / "adapters" - for name, proto_id in (("gmail", OAUTH2), ("telegram", PNBA), ("twin", OAUTH2)): - source = adapter_repo(tmp_path / "src" / name, name, proto_id) - Repo.clone_from(str(source.working_tree_dir), adapters_dir / f"{name}-id") - # Same platform and protocol as gmail, so it's skipped. - twin = Repo(adapters_dir / "twin-id") - commit_manifest(twin, "gmail", OAUTH2) - (adapters_dir / "not-an-adapter").mkdir() - add_adapter("telegram", PNBA) - - with db.get_session() as session: - imported = manager.import_from_disk(session) - assert [a.name for a in imported] == ["gmail"] - assert imported[0].source_url.endswith("src/gmail") - assert imported[0].commit - - assert _rows() == [("gmail", OAUTH2), ("telegram", PNBA)] - - -@pytest.mark.usefixtures("test_db") -def test_moves_credentials_and_state_out_of_the_code(tmp_path): - add_adapter("gmail", OAUTH2) - add_adapter("telegram", PNBA) - (tmp_path / "adapters/gmail-0").mkdir(parents=True) - (tmp_path / "adapters/gmail-0/credentials.json").write_text("secret") - (tmp_path / "assets/telegram-1").mkdir(parents=True) - (tmp_path / "assets/telegram-1/pending.db").write_text("db") - - with db.get_session() as session: - moved = manager.move_files_out_of_code(session) - assert [a.name for a in moved] == ["gmail", "telegram"] - assert manager.move_files_out_of_code(session) == [] - - assert (tmp_path / "config/gmail-0/credentials.json").read_text() == "secret" - assert (tmp_path / "adapters/gmail-0/credentials.json").is_file() - assert (tmp_path / "state/telegram-1/pending.db").read_text() == "db" - assert not (tmp_path / "assets/telegram-1").exists() - - -@pytest.mark.usefixtures("test_db") -def test_keeps_files_already_out_of_the_code(tmp_path): - add_adapter("gmail", OAUTH2) - for path, text in (("adapters", "old"), ("config", "new")): - (tmp_path / path / "gmail-0").mkdir(parents=True) - (tmp_path / path / "gmail-0/credentials.json").write_text(text) - - with db.get_session() as session: - assert manager.move_files_out_of_code(session) == [] - - assert (tmp_path / "config/gmail-0/credentials.json").read_text() == "new" - - @pytest.mark.usefixtures("test_db") def test_a_concurrent_change_is_a_conflict(): add_adapter("gmail", OAUTH2) diff --git a/tests/scripts/test_build_book.py b/tests/scripts/test_build_book.py index afdddaa..2a7294d 100644 --- a/tests/scripts/test_build_book.py +++ b/tests/scripts/test_build_book.py @@ -9,7 +9,7 @@ @pytest.fixture def build(install, tmp_path): - """Run build-book.sh with stub curl and uname, so nothing is downloaded.""" + """Run build-book.sh with stub curl and uname that download nothing.""" stubs = tmp_path / "bin" stubs.mkdir() for name, body in { @@ -40,13 +40,47 @@ def run(arch: str) -> subprocess.CompletedProcess[str]: return run +@pytest.fixture +def cached_tools(install): + """Cached mdbook and mdbook-mermaid, whose stub writes a bundled mermaid.min.js.""" + tools = install / ".mdbook" + for tool, version, body in ( + ("mdbook", "0.5.3", ""), + ("mdbook-mermaid", "0.17.1", 'echo bundled >"$2/mermaid.min.js"'), + ): + exe = tools / f"{tool}-{version}" / tool + exe.parent.mkdir(parents=True) + exe.write_text(f"#!/usr/bin/env bash\n{body}\n") + exe.chmod(exe.stat().st_mode | stat.S_IEXEC) + return tools + + class TestBuildBook: - def test_refuses_a_download_that_fails_its_checksum(self, install, build): + def test_refuses_an_mdbook_download_that_fails_its_checksum(self, install, build): result = build("x86_64") assert result.returncode != 0 - assert "doesn't match its sha256" in result.stderr + assert "mdbook v0.5.3 download doesn't match its sha256" in result.stderr assert not (install / ".mdbook" / "mdbook-0.5.3" / "mdbook").exists() + def test_refuses_a_mermaid_download_that_fails_its_checksum( + self, cached_tools, build + ): + result = build("x86_64") + + assert result.returncode != 0 + assert "mermaid.min.js v12.2.0 download doesn't match" in result.stderr + assert not list(cached_tools.glob("mermaid-12.2.0.min.js*")) + + def test_uses_the_cached_pinned_mermaid_over_the_bundled_one( + self, install, cached_tools, build + ): + (cached_tools / "mermaid-12.2.0.min.js").write_text("pinned\n") + + result = build("x86_64") + + assert result.returncode == 0, result.stderr + assert (install / "mermaid.min.js").read_text() == "pinned\n" + def test_refuses_an_architecture_without_a_release(self, tmp_path, build): result = build("riscv64") assert result.returncode != 0 diff --git a/tests/scripts/test_lib.py b/tests/scripts/test_lib.py index 59bd7c0..36aea53 100644 --- a/tests/scripts/test_lib.py +++ b/tests/scripts/test_lib.py @@ -44,7 +44,7 @@ def test_missing_key_or_file_is_empty(self, bash): class TestPrompt: @pytest.mark.parametrize("name", ["reply", "question", "default"]) def test_without_a_terminal_sets_the_default(self, bash, name): - # setsid drops the controlling terminal, so nothing can answer. + # setsid runs the prompt without a controlling terminal. code = 'source scripts/lib.sh; prompt "$0" "Q? " fallback; echo "${!0}"' result = bash(f"setsid -w bash -c '{code}' {name} str: - result = bash("bash scripts/migrate-runtime-data.sh") - assert result.returncode == 0, result.stderr - return result.stdout - - return run - - -@pytest.fixture -def legacy(install): - """An install laid out the way it was before runtime data moved to data/.""" - (install / "platforms/adapters/demo").mkdir(parents=True) - (install / "platforms/adapters/demo/main.py").touch() - (install / "platforms/adapters_venv/demo/bin").mkdir(parents=True) - (install / "platforms/adapters_assets").mkdir() - (install / "gateway_clients").mkdir() - (install / "gateway_clients/registry.json").write_text("{}") - (install / ".env").write_text(LEGACY_ENV.format(install=install)) - return install - - -def env_value(install, key): - lines = (install / ".env").read_text().splitlines() - return next(line.split("=", 1)[1] for line in lines if line.startswith(f"{key}=")) - - -class TestMigrateRuntimeData: - def test_moves_legacy_layout_into_data(self, legacy, migrate): - migrate() - - assert (legacy / "data/platforms/adapters/demo/main.py").is_file() - assert (legacy / "data/platforms/venvs/demo").is_dir() - assert (legacy / "data/platforms/state").is_dir() - assert (legacy / "data/gateway_clients/registry.json").is_file() - assert not (legacy / "platforms/adapters").exists() - - def test_rewrites_env_keeping_relative_or_absolute_form(self, legacy, migrate): - migrate() - - assert env_value(legacy, "PLATFORMS_ADAPTERS_DIR") == "data/platforms/adapters" - assert ( - env_value(legacy, "PLATFORMS_ADAPTERS_VENV_DIR") - == f"{legacy}/data/platforms/venvs" - ) - - def test_moves_default_dirs_when_env_leaves_them_unset(self, legacy, migrate): - (legacy / ".env").write_text("") - - migrate() - - assert (legacy / "data/platforms/adapters/demo").is_dir() - assert (legacy / ".env").read_text() == "" - - def test_leaves_custom_locations_alone(self, legacy, migrate): - (legacy / ".env").write_text("PLATFORMS_ADAPTERS_DIR=/srv/adapters\n") - - migrate() - - assert (legacy / "platforms/adapters/demo").is_dir() - assert env_value(legacy, "PLATFORMS_ADAPTERS_DIR") == "/srv/adapters" - - def test_rerun_changes_nothing(self, legacy, migrate): - migrate() - env = (legacy / ".env").read_text() - - assert "Moved" not in migrate() - assert (legacy / ".env").read_text() == env - - def test_points_venv_scripts_at_new_paths(self, legacy, migrate): - old_venv = legacy / "platforms/adapters_venv/demo" - (old_venv / "bin/pip").write_text(f"#!{old_venv}/bin/python\n") - - migrate() - - pip = (legacy / "data/platforms/venvs/demo/bin/pip").read_text() - assert pip == f"#!{legacy}/data/platforms/venvs/demo/bin/python\n" - - def test_renames_assets_to_state(self, legacy, migrate): - (legacy / "data/platforms/assets/demo").mkdir(parents=True) - (legacy / "data/platforms/assets/demo/pending.db").write_text("x") - (legacy / ".env").write_text( - "PLATFORMS_ADAPTERS_ASSETS_DIR=data/platforms/assets\n" - ) - - migrate() - - assert (legacy / "data/platforms/state/demo/pending.db").read_text() == "x" - assert not (legacy / "data/platforms/assets").exists() - assert ( - env_value(legacy, "PLATFORMS_ADAPTERS_STATE_DIR") == "data/platforms/state" - ) - - @pytest.mark.parametrize( - ("overrides", "copied"), [('[{"mcc": "624"}]', True), ("[]", False)] - ) - def test_copies_only_local_mcc_mnc_overrides( - self, legacy, migrate, overrides, copied - ): - (legacy / "gateway_clients/mcc_mnc_overrides.json").write_text(overrides) - - migrate() - - assert ( - legacy / "data/gateway_clients/mcc_mnc_overrides.json" - ).exists() is copied diff --git a/tests/scripts/test_sync_env.py b/tests/scripts/test_sync_env.py index 1a43874..0df7030 100644 --- a/tests/scripts/test_sync_env.py +++ b/tests/scripts/test_sync_env.py @@ -103,7 +103,7 @@ def test_without_yes_and_no_terminal_nothing_is_written(self, install, bash): (install / "template.env").write_text(TEMPLATE) env = install / ".env" env.write_text("# Server\nHOST=0.0.0.0\n") - # setsid drops the controlling terminal, so the prompt gets no answer. + # setsid runs the prompt without a controlling terminal. run = 'setsid -w bash "{}/sync-env.sh" {} template.env MailMessage: - """Build an imap_tools MailMessage with a UID from raw RFC822 bytes. - - Matches what a real IMAP fetch produces, without an IMAP connection. - """ + """Build an imap_tools MailMessage with a UID from raw RFC822 bytes.""" headers = [] if from_addr is not None: headers.append(f"From: {from_addr}") @@ -33,7 +30,7 @@ def build_email(body: str, from_addr: str | None = "user@example.com") -> MailMe @pytest.fixture(autouse=True) def _default_auth_allow(monkeypatch): - """Let the sender and auth checks pass, so tests override only what they test.""" + """Pass the sender, auth and payload checks and stub out queueing.""" monkeypatch.setattr(smtp_listener.smtp_auth, "is_sender_allowed", lambda addr: True) monkeypatch.setattr( smtp_listener.smtp_auth, "evaluate", lambda msg, raw, addr: (True, "ok") @@ -114,4 +111,3 @@ def _raise(_addr): monkeypatch.setattr(smtp_listener.smtp_auth, "is_sender_allowed", _raise) msg = build_email(json.dumps({"address": "+1", "text": "dGVzdA=="})) assert smtp_listener.process_incoming_email(msg) is False - publication_task.publish_message.delay.assert_not_called() diff --git a/tests/summary.py b/tests/summary.py index 285a893..e44e334 100644 --- a/tests/summary.py +++ b/tests/summary.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-3.0-only """Writes a results table to the GitHub Actions job summary. -pyproject.toml loads it into every pytest run; it does nothing outside GitHub -Actions. It only imports pytest, since the e2e job installs nothing else. +pyproject.toml loads it into every pytest run. It does nothing outside GitHub +Actions and must import only pytest, the one package the e2e job installs. Rows group by test directory, or by what a test module names in SUMMARY_BY: "test" for the test function, or a parametrized fixture such as "dialect". diff --git a/tests/tasks/test_cleanup_task.py b/tests/tasks/test_cleanup_task.py index 616153e..6e0e80a 100644 --- a/tests/tasks/test_cleanup_task.py +++ b/tests/tasks/test_cleanup_task.py @@ -10,7 +10,9 @@ from publisher import db as db_module from publisher.models import platform_adapter as platform_adapters from publisher.models.client_ephemeral_key import ClientEphemeralKey -from publisher.models.platform_adapter import OAUTH2, PNBA +from publisher.models.payload_session import PayloadSession +from publisher.models.platform_adapter import OAUTH2 +from publisher.models.publication_stats import PublicationStats from publisher.models.server_ephemeral_key import ServerEphemeralKey from publisher.models.token import Token from publisher.models.token import create as create_token @@ -30,7 +32,7 @@ def _in_memory_db(): @pytest.fixture(autouse=True) def lookups(monkeypatch): - """No adapter is installed, so upstream revokes are skipped.""" + """Report every adapter as not installed.""" lookup = MagicMock(side_effect=NotImplementedError("no adapter")) monkeypatch.setattr(platform_adapters, "get_for_protocol", lookup) return lookup @@ -112,19 +114,6 @@ def test_cleanup_idle_tokens_cascades_ephemeral_key_deletes(lookups): assert session.scalars(select(ClientEphemeralKey)).all() == [] -def test_cleanup_idle_tokens_attempts_pnba_revoke(lookups): - with db_module.get_session() as session: - idle_token, _ = _make_token( - session, platform="rmail", proto_id=PNBA, created_days_ago=100 - ) - idle_id = idle_token.id - - cleanup_task.cleanup_idle_tokens() - - assert idle_id not in _remaining_token_ids() - lookups.assert_called_once_with(ANY, "rmail", PNBA, include_disabled=True) - - def test_cleanup_idle_tokens_noop_when_nothing_idle(lookups, caplog): with db_module.get_session() as session: _make_token(session, platform="gmail", proto_id=OAUTH2, created_days_ago=0) @@ -134,3 +123,22 @@ def test_cleanup_idle_tokens_noop_when_nothing_idle(lookups, caplog): assert "No idle tokens to clean up" in caplog.text lookups.assert_not_called() + + +def test_cleanup_stale_payload_sessions_counts_each_as_a_failure(): + started = datetime.datetime.now(datetime.UTC) - datetime.timedelta(hours=4) + with db_module.get_session() as session: + session.add_all( + [ + PayloadSession(sender_id="stale", session_id=1, created_at=started), + PayloadSession(sender_id="waiting", session_id=2), + ] + ) + + cleanup_task.cleanup_stale_payload_sessions() + + with db_module.get_session() as session: + assert session.scalars(select(PayloadSession.sender_id)).all() == ["waiting"] + [stat] = session.scalars(select(PublicationStats)).all() + assert (stat.status, stat.failure_reason) == ("failed", "segments_missing") + assert stat.created_at == started diff --git a/tests/tasks/test_publication_task.py b/tests/tasks/test_publication_task.py index 76ff9ce..961560f 100644 --- a/tests/tasks/test_publication_task.py +++ b/tests/tasks/test_publication_task.py @@ -39,7 +39,7 @@ def _patch_infra(monkeypatch): def _stub_publications(monkeypatch, *, prepare_error=None, send_error=None): - """Stub each publication step; an error makes that step raise it.""" + """Stub each publication step, making prepare or send raise the given error.""" stub = MagicMock() stub.validate.return_value = (b"raw", b"seg", object()) stub.prepare.return_value = MagicMock(platform="gmail") @@ -67,19 +67,14 @@ def _run_with_publish_error(monkeypatch, error): [ (PayloadNotSupportedError("unsupported type"), "payload_not_supported"), (PayloadMalformedError("bad payload"), "payload_malformed"), - (ProtocolNotAllowedError("protocol not allowed"), "protocol_not_allowed"), - (OfflineTagMissingError("missing tag"), "offline_tag_missing"), - (OfflineTagInvalidError("invalid tag"), "offline_tag_invalid"), (KeyNotFoundError("Token not found"), "token_not_found"), (KeyUnavailableError("key used"), "key_unavailable"), (TokenVerificationError("unknown key"), "token_verification_failed"), ], ) -def test_pipeline_errors_are_caught_and_logged(monkeypatch, caplog, error, reason): - """The task must swallow these, not raise; the REST caller already got its 200.""" - stub = _run_with_publish_error(monkeypatch, error) +def test_pipeline_errors_are_recorded_as_failed(monkeypatch, caplog, error, reason): + _run_with_publish_error(monkeypatch, error) - stub.send.assert_not_called() assert "Failed to process payload" in caplog.text publication_task.record_publication.assert_called_once() kwargs = publication_task.record_publication.call_args.kwargs @@ -87,6 +82,22 @@ def test_pipeline_errors_are_caught_and_logged(monkeypatch, caplog, error, reaso assert kwargs["failure_reason"] == reason +@pytest.mark.parametrize( + "error, reason", + [ + (ProtocolNotAllowedError("protocol not allowed"), "protocol_not_allowed"), + (OfflineTagMissingError("missing tag"), "offline_tag_missing"), + (OfflineTagInvalidError("invalid tag"), "offline_tag_invalid"), + ], +) +def test_offline_refusals_are_rejected_not_failed(monkeypatch, caplog, error, reason): + _run_with_publish_error(monkeypatch, error) + + assert "Rejected payload" in caplog.text + kwargs = publication_task.record_publication.call_args.kwargs + assert (kwargs["status"], kwargs["failure_reason"]) == ("rejected", reason) + + def test_a_subclass_of_a_mapped_error_keeps_its_code(monkeypatch): class TruncatedPayloadError(PayloadMalformedError): pass @@ -114,14 +125,12 @@ def test_adapter_failures_record_a_code_not_the_message(monkeypatch, code, reaso assert kwargs["failure_reason"] == reason -def test_a_failed_send_rolls_back_and_keeps_the_refreshed_token(monkeypatch, caplog): +def test_a_failed_send_keeps_the_refreshed_token(monkeypatch, caplog): error = AdapterIntegrationError("boom", code=UpstreamError.code, token={"t": 2}) stub = _stub_publications(monkeypatch, send_error=error) publication_task.publish_message("text", "sender", "https") - # The keys' deletion is rolled back; the failure is recorded afterwards. - assert EVENTS == ["open", "send", "rollback", "open", "commit"] assert "Failed to publish message" in caplog.text assert stub.store_token.call_args.args[2] == {"t": 2} stub.finish.assert_not_called() @@ -130,7 +139,6 @@ def test_a_failed_send_rolls_back_and_keeps_the_refreshed_token(monkeypatch, cap def test_unexpected_error_is_caught_logged_and_recorded(monkeypatch, caplog): - """A bare, unanticipated exception must not crash the worker.""" _run_with_publish_error(monkeypatch, RuntimeError("boom")) assert "unexpected error" in caplog.text.lower() @@ -146,6 +154,7 @@ def test_success_records_published_stat(monkeypatch): publication_task.publish_message("text", "sender", "https", country_code="CM") assert stub.prepare.call_args.kwargs["sender_id"] == "sender" + assert EVENTS == ["open", "commit", "send", "open", "commit"] stub.finish.assert_called_once() publication_task.record_publication.assert_called_once() kwargs = publication_task.record_publication.call_args.kwargs @@ -155,7 +164,6 @@ def test_success_records_published_stat(monkeypatch): def test_incomplete_segment_session_skips_recording(monkeypatch): - """Return early without recording an outcome while awaiting more segments.""" stub = _stub_publications(monkeypatch) stub.prepare.return_value = None @@ -165,21 +173,26 @@ def test_incomplete_segment_session_skips_recording(monkeypatch): publication_task.record_publication.assert_not_called() -def test_a_successful_send_commits_once(monkeypatch): - _stub_publications(monkeypatch) +def test_a_recording_failure_after_the_send_is_not_a_failed_publication( + monkeypatch, caplog +): + stub = _stub_publications(monkeypatch) + stub.finish.side_effect = RuntimeError("db dropped") publication_task.publish_message("text", "sender", "https") - assert EVENTS == ["open", "send", "commit"] + assert EVENTS == ["open", "commit", "send", "open", "rollback"] + assert "couldn't record the result" in caplog.text + publication_task.record_publication.assert_not_called() -def test_a_rejected_payload_rolls_back_before_sending(monkeypatch): +def test_a_failed_payload_rolls_back_before_sending(monkeypatch): _run_with_publish_error(monkeypatch, PayloadMalformedError("bad payload")) assert EVENTS == ["open", "rollback", "open", "commit"] -def test_tag_is_forwarded_to_service_publish(monkeypatch): +def test_tag_is_passed_to_prepare(monkeypatch): stub = _stub_publications(monkeypatch) publication_task.publish_message("text", "sender", "https", "s3cret-tag") diff --git a/tests/test_config.py b/tests/test_config.py index 524ecb8..ba63b47 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -64,7 +64,7 @@ def test_check_reports_every_section_together(): @pytest.mark.parametrize("value", ["zz" * 32, "ab" * 16]) -def test_invalid_key_is_rejected(value): +def test_invalid_data_encryption_key_is_rejected(value): assert "DATA_ENCRYPTION_KEY" in errors(DATA_ENCRYPTION_KEY=value) @@ -227,8 +227,6 @@ def test_template_env_paths_match_defaults(): def test_template_env_lists_every_variable_config_reads(): - # sync-env.sh --prune drops what the template lacks, so a variable config - # reads must be in it, if only commented out. template = template_env_names() # An empty environment that records every name config asks for. env = Mock() @@ -249,7 +247,7 @@ def test_template_env_lists_every_variable_config_reads(): def test_settings_reference_matches_template_env(): template = template_env_names() reference = (config.ROOT / "docs" / "configuration" / "settings.md").read_text() - # Only the first column, so a name mentioned in a description doesn't count. + # Names in the first column only, not ones a description mentions. documented = { name for row in re.findall(r"^\|\s*(`[A-Z].*?)\s*\|", reference, re.M) diff --git a/tests/test_publications.py b/tests/test_publications.py index ea0ee72..acb4c3a 100644 --- a/tests/test_publications.py +++ b/tests/test_publications.py @@ -24,6 +24,7 @@ PayloadMalformedError, ProtocolNotAllowedError, ) +from publisher.tasks import publication_task from relaysms_adapter_sdk import Account, Message, UpstreamError from tests.helpers import add_adapter @@ -117,7 +118,7 @@ def test_https_offline_unchecked_when_secret_unset(offline): assert result == "rmail" -def test_protocol_allowlist_is_still_enforced_before_tag_check(set_config, offline): +def test_protocol_allowlist_applies_even_with_a_correct_tag(set_config, offline): set_config( publications, "offline_config", @@ -132,10 +133,6 @@ def test_protocol_allowlist_is_still_enforced_before_tag_check(set_config, offli def test_online_payload_bypasses_protocol_and_tag_checks( set_config, monkeypatch, offline ): - """Skip the offline-only allowlist and tag checks for online payloads. - - Holds even over https with a secret configured and no tag. - """ set_config( publications, "offline_config", @@ -153,7 +150,7 @@ def test_online_payload_bypasses_protocol_and_tag_checks( ) -# Publishing end to end: real keys and ciphertext; only the adapter process is faked. +# Publishing end to end with real keys and ciphertext. Only the adapter is faked. SENDER = "+237600000000" # A 1x1 PNG, padded so the payload spans several SMS segments. @@ -194,7 +191,7 @@ def account(test_db): ) return SimpleNamespace( token_id=token.token_id, - # The token's own slot is used up; any other in 16-255 is free. + # Any slot in 16-255 but the token's own is free. slot=17 if token_slot != 17 else 18, client_keys=client_keys, server_keys=server_keys, @@ -251,21 +248,14 @@ def _prepare(text, protocol="sms"): def _publish(text, protocol="sms"): - """Publish in one transaction, as the task does; return the platform.""" - payload_raw, raw_segment, payload_type = publications.validate(text) + """Publish and return the platform, or None while segments are missing.""" + delivery = _prepare(text, protocol) + if delivery is None: + return None + new_token = publications.send(delivery) with get_session() as s: - delivery = publications.prepare( - s, - payload_raw=payload_raw, - sender_id=SENDER, - raw_segment=raw_segment, - payload_type=payload_type, - protocol=protocol, - ) - if delivery is None: - return None - publications.finish(s, delivery, publications.send(delivery)) - return delivery.platform + publications.finish(s, delivery, new_token) + return delivery.platform def _slot_has_keys(account, slot): @@ -333,18 +323,15 @@ def test_publish_saves_a_refreshed_token(account, adapter): assert _stored_token(account)[0]["token"] == refreshed -def test_a_failed_send_keeps_the_keys_for_a_retry(account, adapter): +def test_a_failed_send_cant_publish_again(account, adapter): adapter.results["send_message"] = UpstreamError("platform down") contents = _encrypt(account, account.slot, _email()) [text] = _sms(contents, slot=account.slot, token_id=account.token_id) - with pytest.raises(AdapterIntegrationError): - _publish(text) - assert _slot_has_keys(account, account.slot) - + publication_task.publish_message(text, SENDER) adapter.results["send_message"] = {} - assert _publish(text) == "gmail" - assert not _slot_has_keys(account, account.slot) + publication_task.publish_message(text, SENDER) + assert len(adapter.calls) == 1 def test_a_failed_send_still_saves_the_refreshed_token(account, adapter): @@ -352,13 +339,8 @@ def test_a_failed_send_still_saves_the_refreshed_token(account, adapter): adapter.results["send_message"] = UpstreamError("post failed", token=refreshed) contents = _encrypt(account, account.slot, _email()) [text] = _sms(contents, slot=account.slot, token_id=account.token_id) - delivery = _prepare(text) - assert delivery - with pytest.raises(AdapterIntegrationError) as error: - publications.send(delivery) - with get_session() as s: - publications.store_token(s, delivery, error.value.token) + publication_task.publish_message(text, SENDER) assert _stored_token(account)[0]["token"] == refreshed @@ -390,6 +372,16 @@ def test_attachment_is_published_once_all_segments_arrive(account, adapter, capl assert attached.data == PNG +def test_a_segment_delivered_twice_is_joined_once(account, adapter): + contents = _encrypt(account, 5, _email(attachment=PNG)) + texts = _sms(contents, slot=5, token_id=account.token_id, attachment=PNG, sess_id=3) + + results = [_publish(text) for text in [texts[0], *texts]] + + assert results[-1] == "gmail" + assert len(adapter.calls) == 1 + + def test_offline_payload_goes_to_the_offline_adapter(test_db, adapter): with get_session() as s: keys.initialize_server_identity_keys(s) diff --git a/tools/client.py b/tools/client.py index 56a66b0..9796ecb 100755 --- a/tools/client.py +++ b/tools/client.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-only -"""CLI tool for testing gRPC flows.""" +"""Reference client for the gRPC and REST flows.""" import json import logging @@ -79,7 +79,7 @@ def shared_options(f): """ ) def cli(): - """CLI test tool for gRPC flows.""" + """Reference client for the gRPC and REST flows.""" pass @@ -440,7 +440,7 @@ def cmd_exchange_pnba_code( **_, ): """Exchange a PNBA code, decrypt the token, and store session data.""" - logger.info("platform=%s | phone=%s | code=%s", platform, phone_number, code) + logger.info("platform=%s", platform) client_keypairs = [X25519PrivateKey.generate() for _ in range(256)] @@ -781,7 +781,7 @@ def cmd_sync_keys(host, port, tls, rest_api, token, **_): "--address", required=True, metavar="MSISDN", - help="Sender's phone number (e.g. +237123456789).", + help="Sender's phone number (e.g. +237600000000).", ) @click.option("--to", metavar="ADDRESS", help="Recipient address (email or phone).") @click.option( @@ -843,7 +843,7 @@ def cmd_send( if offline: if not platform or platform.lower() != "rmail": - logger.error("--offline is currently only supported for --platform rmail.") + logger.error("--offline supports only --platform rmail.") sys.exit(1) if not to: logger.error("--to is required when using --offline.") @@ -924,7 +924,7 @@ def cmd_send( resp = requests.post(url, json=req_body, timeout=30) resp.raise_for_status() except requests.exceptions.HTTPError as e: - logger.error("HTTP error on seg %d: %s -- %s", idx, e, resp.text) + logger.error("HTTP error on seg %d: %s: %s", idx, e, resp.text) sys.exit(1) except Exception as e: logger.error("request failed on seg %d: %s", idx, e) diff --git a/tools/client_helpers.py b/tools/client_helpers.py index 4ec4494..c47983e 100644 --- a/tools/client_helpers.py +++ b/tools/client_helpers.py @@ -17,7 +17,6 @@ logger = logging.getLogger(__name__) -# Beside the client, so it works from any working directory. DB_PATH = Path(__file__).with_name("db.json") @@ -70,7 +69,7 @@ def grpc_call(fn, *args, **kwargs) -> tuple: try: return fn(*args, **kwargs), None except grpc.RpcError as e: - return None, f"gRPC error: {e.code()} -- {e.details()}" + return None, f"gRPC error: {e.code()}: {e.details()}" def fetch_server_identity_public_key(rest_api_url: str, key_id: int) -> bytes: @@ -86,7 +85,7 @@ def fetch_server_identity_public_key(rest_api_url: str, key_id: int) -> bytes: def build_v1_request_metadata( rest_api: str, method_name: str, payload: bytes | None = None ) -> tuple[bytes, bytes, list[tuple]]: - """Encrypt a request to a random server identity key; return its headers.""" + """Return the client key pair and headers for an encrypted request.""" key_id = secrets.randbelow(256) ss_pk_bytes = fetch_server_identity_public_key(rest_api, key_id)