From 9d20e174b492ef404bc883c6254957c65aace6d4 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Thu, 22 Oct 2026 14:37:40 +0100 Subject: [PATCH 01/13] docs: reword vague comments and docstrings to state the condition and outcome --- publisher/api/grpc/server.py | 5 +++-- publisher/api/grpc/utils.py | 4 ++-- publisher/api/rest/app.py | 2 +- publisher/api/rest/v1/auth.py | 3 ++- publisher/api/rest/v1/creds.py | 2 +- publisher/api/rest/v1/params.py | 3 ++- publisher/api/rest/v1/platform_adapters.py | 7 ++++--- publisher/api/rest/v1/platforms.py | 2 -- publisher/api/rest/v1/schemas.py | 3 ++- publisher/cli/seed.py | 3 ++- publisher/credentials.py | 3 ++- publisher/db/__init__.py | 3 ++- publisher/gateway_clients/mcc_mnc.py | 5 +++-- publisher/models/credential_session.py | 3 ++- publisher/models/platform_adapter_job.py | 7 ++++--- publisher/platforms/adapter_config.py | 7 ++++--- publisher/platforms/manager.py | 9 ++++++--- publisher/publications.py | 5 +++-- publisher/tasks/publication_task.py | 7 ++++--- publisher/tokens.py | 8 ++++---- 20 files changed, 53 insertions(+), 38 deletions(-) diff --git a/publisher/api/grpc/server.py b/publisher/api/grpc/server.py index 94a67c8..cdecb53 100644 --- a/publisher/api/grpc/server.py +++ b/publisher/api/grpc/server.py @@ -78,7 +78,7 @@ def _bind_port(grpc_server: grpc.Server) -> None: return cert_file, key_file = grpc_config.tls_cert_file, grpc_config.tls_key_file - assert cert_file and key_file # config requires both when TLS is on + assert cert_file and key_file # config refuses TLS without both files try: credentials = _load_ssl_credentials(Path(cert_file), Path(key_file)) except FileNotFoundError as e: @@ -105,7 +105,8 @@ def serve() -> None: grpc_server = _build_server(grpc_config.max_workers) _bind_port(grpc_server) - # Stopping inside the handler hangs if a second signal arrives mid-stop. + # The handler only sets this flag: stopping inside it would hang if a second + # signal arrived while the server was stopping. stop_requested = threading.Event() def request_stop(signum: int, _frame) -> None: diff --git a/publisher/api/grpc/utils.py b/publisher/api/grpc/utils.py index 8ab9354..1b27d21 100644 --- a/publisher/api/grpc/utils.py +++ b/publisher/api/grpc/utils.py @@ -55,7 +55,6 @@ def validate_client_ephemeral_public_keys(keys) -> None: def find_adapter(platform: str, proto_id: int) -> PlatformAdapter: - # Its own session, so none stays open while the adapter runs. with get_session() as s: return platform_adapters.get_for_protocol(s, platform, proto_id) @@ -63,7 +62,8 @@ def find_adapter(platform: str, proto_id: int) -> PlatformAdapter: def call_adapter(adapter: PlatformAdapter, method: str, request: Any) -> dict: """Call an adapter, turning its errors into gRPC errors. - Rejected input and codes reach the client; anything else is logged. + Invalid input, failed authentication and rate limits reach the client with the + adapter's message; any other error is logged and the client gets a generic one. """ try: return ipc.call(adapter, method, request) diff --git a/publisher/api/rest/app.py b/publisher/api/rest/app.py index 534e014..3b6e833 100644 --- a/publisher/api/rest/app.py +++ b/publisher/api/rest/app.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -# Query and path values are already in the URL; body values can be secrets. +# Validation errors echo values from these locations only. ECHOED_INPUT_LOCATIONS = frozenset({"query", "path"}) MAX_ECHOED_INPUT_LENGTH = 50 diff --git a/publisher/api/rest/v1/auth.py b/publisher/api/rest/v1/auth.py index 04e593f..4abab83 100644 --- a/publisher/api/rest/v1/auth.py +++ b/publisher/api/rest/v1/auth.py @@ -105,7 +105,8 @@ def _request_origin(request: Request) -> str | None: def _record_failed_login(db: Session, username: str, method: str) -> None: - # Not for unknown usernames: anyone can send those, so they'd flood the table. + # Recorded only for existing usernames: anyone can send unknown ones, which + # would fill the audit table. target = get_by_username(db, username) if target is None: return diff --git a/publisher/api/rest/v1/creds.py b/publisher/api/rest/v1/creds.py index ce8de79..0eaecaf 100644 --- a/publisher/api/rest/v1/creds.py +++ b/publisher/api/rest/v1/creds.py @@ -72,7 +72,7 @@ def _model_errors( log=f"{by}: {e}", ) from None except CredentialPermissionError as e: - # Keep the denial, not the change. + # Undo the refused change, then record and commit the denial. db.rollback() audit_event.record( db, diff --git a/publisher/api/rest/v1/params.py b/publisher/api/rest/v1/params.py index f02cc0d..be7b307 100644 --- a/publisher/api/rest/v1/params.py +++ b/publisher/api/rest/v1/params.py @@ -44,7 +44,8 @@ def check_if_match(request: Request, current: str, what: str) -> None: if_match = request.headers.get("If-Match") if if_match is None: raise ApiError(428, "If-Match header required.") - # nginx gzip turns our ETag weak (W/"..."), and ours are version-based anyway. + # nginx gzip marks the ETag weak (W/"..."). Ours come from the row version, not + # the body, so a weak one still matches. if if_match.strip().removeprefix("W/") != current: raise ApiError( 412, diff --git a/publisher/api/rest/v1/platform_adapters.py b/publisher/api/rest/v1/platform_adapters.py index b7c3d6a..8892bdd 100644 --- a/publisher/api/rest/v1/platform_adapters.py +++ b/publisher/api/rest/v1/platform_adapters.py @@ -105,7 +105,6 @@ def _adapter_errors(context: AuthContext): except (AdapterInUseError, AdapterConflictError) as e: raise ApiError(409, str(e), log=f"{by}: {e}") from None except AdapterFaultError as e: - # Its message can name files on this server. message = "The adapter or its files are broken; see the server log." raise ApiError(502, message, log=f"{by}: {e}") from None except AdapterError as e: @@ -292,7 +291,8 @@ def setup_config( _require_administrator(context, "Adapter settings") adapter = _load(db, adapter_id) check_if_match(request, _etag(adapter), "adapter") - # End the read before the adapter runs; the version still catches a change. + # End the read transaction before the adapter runs. A change made meanwhile + # still fails the save, which checks the adapter's version. db.commit() with _adapter_errors(context): adapter_config.setup(db, adapter, body.inputs, actor=context.credential) @@ -320,7 +320,8 @@ def _queue( try: run_adapter_job.delay(str(job.id)) except Exception as e: - # Fail it now, or its lock would block the adapter until cleanup. + # Fail the job now. Otherwise it stays active and blocks every other job for + # the adapter until cleanup fails it as abandoned. jobs.finish(db, job.id, state="failed", log=[f"Queueing failed: {e}"]) db.commit() raise ApiError( diff --git a/publisher/api/rest/v1/platforms.py b/publisher/api/rest/v1/platforms.py index 12fe02e..fa4eca1 100644 --- a/publisher/api/rest/v1/platforms.py +++ b/publisher/api/rest/v1/platforms.py @@ -88,8 +88,6 @@ def get_platform_oauth_client_metadata( detail="OAuth client metadata file not found for this platform", ) from exc except ValueError as exc: - # The error names the file's path on this server, so the client only gets - # a generic message. logger.error("Invalid OAuth client metadata: %s", exc) raise HTTPException( status_code=500, detail="OAuth client metadata is invalid" diff --git a/publisher/api/rest/v1/schemas.py b/publisher/api/rest/v1/schemas.py index ca3ce23..e11dc8a 100644 --- a/publisher/api/rest/v1/schemas.py +++ b/publisher/api/rest/v1/schemas.py @@ -42,7 +42,8 @@ class OAuthClientMetadata(BaseModel): client_name: str client_uri: str scope: str - # The same for every atproto web client, so settings don't include them. + # The same for every atproto web client, so the adapter's settings leave them + # out and these defaults fill them in. dpop_bound_access_tokens: bool = True application_type: str = "web" grant_types: list[str] = ["authorization_code", "refresh_token"] diff --git a/publisher/cli/seed.py b/publisher/cli/seed.py index feac9ec..237af43 100644 --- a/publisher/cli/seed.py +++ b/publisher/cli/seed.py @@ -173,7 +173,8 @@ def gateway_clients_command(count): while added < count: msisdn = _random_mobile(random.choice(COUNTRIES)) suggestion = gateway_clients.suggest(msisdn) - # Picks for the ambiguous ones, as an administrator would. + # Takes the first candidate when the operator is ambiguous, as an + # administrator would. candidate = suggestion.candidates[0] try: client = gateway_clients.create( diff --git a/publisher/credentials.py b/publisher/credentials.py index b504174..082484d 100644 --- a/publisher/credentials.py +++ b/publisher/credentials.py @@ -33,7 +33,8 @@ password_hasher = PasswordHasher() -# Each scope here can only be granted with the scope it maps to. +# Granting the first scope of each pair also needs the second, e.g. gc:write needs +# gc:read. _REQUIRES = { Scope.STATS_PUBLICATIONS_REASONS: Scope.STATS_PUBLICATIONS_READ, Scope.GC_WRITE: Scope.GC_READ, diff --git a/publisher/db/__init__.py b/publisher/db/__init__.py index d66bfe1..5960e63 100644 --- a/publisher/db/__init__.py +++ b/publisher/db/__init__.py @@ -161,7 +161,8 @@ def _create_engine() -> Engine: else: _ensure_sqlite_parent_dir(database.sqlite_path) if database.encryption_enabled: - assert database.encryption_key is not None # config requires it here + # config refuses SQLCipher without a key + assert database.encryption_key is not None logger.debug("Using SQLCipher3 encryption for SQLite") from sqlcipher3 import dbapi2 as sqlcipher3_dbapi diff --git a/publisher/gateway_clients/mcc_mnc.py b/publisher/gateway_clients/mcc_mnc.py index a9ce946..a319c41 100644 --- a/publisher/gateway_clients/mcc_mnc.py +++ b/publisher/gateway_clients/mcc_mnc.py @@ -30,7 +30,8 @@ ) # Snapshot rows that aren't operators. _NOT_OPERATORS = frozenset({"", "failed calls", "fix line", "unknown"}) -# A word in this many of a region's networks only weakly tells them apart. +# A word in at least this many of a region's network names is common, so sharing +# it is only a weak match. _COMMON_IN_REGION = 3 @@ -85,7 +86,7 @@ def _score( ): best = max(best, 2) elif shared: - best = max(best, 1) # Only words many of the region's networks have. + best = max(best, 1) # Only common words are shared. return best diff --git a/publisher/models/credential_session.py b/publisher/models/credential_session.py index 7c5732b..9d6a2ee 100644 --- a/publisher/models/credential_session.py +++ b/publisher/models/credential_session.py @@ -106,7 +106,8 @@ def get_active(session: Session, raw_token: str) -> CredentialSession | None: return None credential = credential_session.credential - # None if a login raced a delete, since sqlite doesn't enforce the foreign key. + # None when the credential was deleted while this session was being created, + # since SQLite doesn't enforce the foreign key. if credential is None or not credential.is_active: return None if credential_session.session_version != credential.session_version: diff --git a/publisher/models/platform_adapter_job.py b/publisher/models/platform_adapter_job.py index 5fe08b5..d5625a2 100644 --- a/publisher/models/platform_adapter_job.py +++ b/publisher/models/platform_adapter_job.py @@ -35,8 +35,8 @@ class PlatformAdapterJob(Base): from_commit: Mapped[str | None] = mapped_column(String(40), default=None) to_commit: Mapped[str | None] = mapped_column(String(40), default=None) state: Mapped[str] = mapped_column(String(16), default="queued") - # The adapter id while the job is active, NULL after. Being unique, it allows - # one active job per adapter on every database. + # The adapter id while the job is queued or running, NULL once it finishes. + # Its unique index allows one active job per adapter. lock: Mapped[str | None] = mapped_column(String(36), default=None) log: Mapped[str] = mapped_column(Text, default="") requested_by: Mapped[uuid.UUID | None] = mapped_column(Uuid, default=None) @@ -159,7 +159,8 @@ def delete_older_than(session: Session, cutoff: datetime.datetime) -> int: result = cast( CursorResult, session.execute( - # By finish time, so a job failed just now as abandoned stays readable. + # By finished_at, so an old job that cleanup just failed as abandoned + # is kept for the full retention period. delete(PlatformAdapterJob).where(PlatformAdapterJob.finished_at < cutoff) ), ) diff --git a/publisher/platforms/adapter_config.py b/publisher/platforms/adapter_config.py index f00c224..de1f93c 100644 --- a/publisher/platforms/adapter_config.py +++ b/publisher/platforms/adapter_config.py @@ -163,13 +163,14 @@ def _save( actor: Credential | None, changed: dict[str, Any], ) -> None: - # Touching the row moves its version on, so a concurrent change conflicts. + # Updating the row bumps its version, so a concurrent change to the adapter + # fails with a conflict. adapter.updated_at = utc_now() adapter.updated_by = actor.id if actor else None flush(session, adapter) audit_event.record( session, action, actor=actor, target=adapter, details={"keys": sorted(changed)} ) - # Written before the commit, unlike manager.activate: the flush holds the row, - # so concurrent saves reach the file in order. + # Written before the commit, unlike manager.activate: the flush locks the row + # until the commit, so concurrent saves write the file one at a time. write_config(settings, directory=Path(adapter.config_path)) diff --git a/publisher/platforms/manager.py b/publisher/platforms/manager.py index 4785fd3..5ed8c27 100644 --- a/publisher/platforms/manager.py +++ b/publisher/platforms/manager.py @@ -244,7 +244,8 @@ def _register(session: Session, adapter: PlatformAdapter, manifest: Manifest) -> def activate(adapter: PlatformAdapter) -> None: """Move a built version into place. Call it after the install or update commits. - Run before the commit, a failed commit would leave the new files with the old row. + If it ran first and the commit failed, the new files would be in place while the + database still names the old version. """ # Renames are atomic, so an adapter call sees the old version or the new. for current in (Path(adapter.path), Path(adapter.venv_path)): @@ -311,7 +312,8 @@ def update( """Register tag, or the newest version tag, for an adapter.""" tag = tag or latest_tag(adapter.source_url) commit, manifest = _build(adapter, tag, log) - # A tag that now names other code was moved, which a release shouldn't do. + # Refused: the installed tag now points to other code, so it was moved after + # its release. if tag == adapter.tag and commit != adapter.commit: _discard_build(adapter) raise AdapterError( @@ -363,7 +365,8 @@ def remove( if adapter.id in ("", ".", "..") or "/" in adapter.id: raise AdapterError(f"Unsafe adapter id {adapter.id!r}") linked = tokens.count_for_platform(session, adapter.name, adapter.proto_id) - # Their tokens can only be revoked upstream through the adapter. + # Refused while accounts are linked: without the adapter, their tokens can't + # be revoked at the platform. if linked and not force: raise AdapterInUseError( f"{linked} linked account(s) use {adapter.name!r}. Disable it instead, " diff --git a/publisher/publications.py b/publisher/publications.py index 39013b2..721ec12 100644 --- a/publisher/publications.py +++ b/publisher/publications.py @@ -84,7 +84,8 @@ def __init__( ): super().__init__(message, platform_name=platform_name) self.code = code - # Refreshed before the send failed, so still the one to store. + # A token the adapter refreshed before the send failed; it replaces the + # stored one. self.token = token @@ -354,7 +355,7 @@ def send(delivery: Delivery) -> dict | None: def store_token(session: Session, delivery: Delivery, new_token: dict | None) -> None: - """Store a token the adapter refreshed, unless it was unlinked meanwhile.""" + """Store a token the adapter refreshed, if the account is still linked.""" token = session.get(Token, delivery.token_id) if delivery.token_id else None if token and new_token and new_token != token.token_data.get("token"): update_token_data(session, token, {**token.token_data, "token": new_token}) diff --git a/publisher/tasks/publication_task.py b/publisher/tasks/publication_task.py index 22d5957..2ff20d8 100644 --- a/publisher/tasks/publication_task.py +++ b/publisher/tasks/publication_task.py @@ -82,8 +82,9 @@ def publish_message( """Validate and publish a payload, then record the outcome.""" delivery = refreshed_token = None try: - # Commits only after a successful send, so a failure leaves the payload's - # keys usable, and a copy of it arriving meanwhile waits on their row lock. + # Commits only after a successful send. A failure rolls back, so the payload's + # keys can be used again, and a duplicate that arrives during the send waits + # on their row lock. with get_session() as db: payload_raw, raw_segment, payload_type = publications.validate(text_payload) delivery = publications.prepare( @@ -96,7 +97,7 @@ def publish_message( tag=tag, ) if delivery is None: - # Incomplete multi-segment session, awaiting more parts. + # Segments of a multi-part payload are still missing. return new_token = publications.send(delivery) publications.finish(db, delivery, new_token) diff --git a/publisher/tokens.py b/publisher/tokens.py index 750f31f..2c93e68 100644 --- a/publisher/tokens.py +++ b/publisher/tokens.py @@ -26,10 +26,10 @@ def account(token: Token) -> Account: def revocation(session: Session, token: Token) -> Revocation | None: - """Read what revoking a token takes, before the token is deleted. + """Return the adapter and request that revoke a token; call it before deleting it. - None when no adapter serves its platform. Run it with revoke_upstream once the - deletion commits. + None when no adapter serves its platform. Pass the result to revoke_upstream + after the deletion commits. """ try: # Disabled adapters still revoke, so users can always unlink. @@ -47,7 +47,7 @@ def revocation(session: Session, token: Token) -> Revocation | None: def revoke_upstream(revocation: Revocation | None) -> None: - """Revoke at the platform, outside any transaction; failures are only logged.""" + """Revoke a token at the platform once its deletion commits; failures are logged.""" if revocation is None: return adapter, request = revocation From 490617d7d920fc36967e538f1482aed6dd36161e Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Thu, 22 Oct 2026 14:38:40 +0100 Subject: [PATCH 02/13] docs(sdk): reword vague docstrings to state the condition and outcome --- sdk/pyproject.toml | 2 +- sdk/src/relaysms_adapter_sdk/adapter.py | 2 +- sdk/src/relaysms_adapter_sdk/errors.py | 5 ++++- sdk/src/relaysms_adapter_sdk/runner.py | 5 +++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index a1ba40d..9320c64 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "relaysms-adapter-sdk" -version = "1.1.2" +version = "1.1.3" description = "Build RelaySMS Publisher platform adapters" readme = "README.md" license = "GPL-3.0-only" diff --git a/sdk/src/relaysms_adapter_sdk/adapter.py b/sdk/src/relaysms_adapter_sdk/adapter.py index d4e30ac..3095be3 100644 --- a/sdk/src/relaysms_adapter_sdk/adapter.py +++ b/sdk/src/relaysms_adapter_sdk/adapter.py @@ -49,7 +49,7 @@ def revoke(self, request: RevokeRequest) -> MaybeAwaitable[None]: def setup(cls, request: SetupRequest) -> MaybeAwaitable[SetupResult]: """Turn the [setup] inputs into settings, e.g. by registering an app. - A classmethod: it runs before the settings an instance reads exist. + A classmethod, since an adapter's __init__ may read the settings setup creates. """ raise AdapterError( f"{cls.__name__} has no setup; set its config instead.", diff --git a/sdk/src/relaysms_adapter_sdk/errors.py b/sdk/src/relaysms_adapter_sdk/errors.py index d791ab6..dc5af2b 100644 --- a/sdk/src/relaysms_adapter_sdk/errors.py +++ b/sdk/src/relaysms_adapter_sdk/errors.py @@ -12,7 +12,10 @@ class AdapterError(Exception): - """A failure the Publisher should report as is.""" + """A failure whose message reaches the Publisher. + + Any other exception reaches it only as INTERNAL_ERROR, with details in the log. + """ code: int = -32000 diff --git a/sdk/src/relaysms_adapter_sdk/runner.py b/sdk/src/relaysms_adapter_sdk/runner.py index bf5a013..ade278d 100644 --- a/sdk/src/relaysms_adapter_sdk/runner.py +++ b/sdk/src/relaysms_adapter_sdk/runner.py @@ -36,8 +36,9 @@ def handle( ) -> str: """Answer one JSON-RPC request line. - Given the class, it creates the adapter for the request, except for setup, - which runs on the class before the settings an adapter reads exist. + Given the class, it creates an adapter for each request, except setup, which + runs on the class because an adapter's __init__ may read the settings setup + creates. """ try: request = wire.parse_request(line) From 26eeb605346c88ad0f664c6b343ec97cee7dab4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:23:47 +0000 Subject: [PATCH 03/13] chore(deps): bump fastapi from 0.143.3 to 0.149.0 in the python group Bumps the python group with 1 update: [fastapi](https://github.com/fastapi/fastapi). Updates `fastapi` from 0.143.3 to 0.149.0 - [Release notes](https://github.com/fastapi/fastapi/releases) - [Commits](https://github.com/fastapi/fastapi/compare/0.143.3...0.149.0) --- updated-dependencies: - dependency-name: fastapi dependency-version: 0.149.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index db42364..32d784f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cachetools==7.2.3 celery[sqlalchemy,redis]==5.6.3 cryptography==50.0.2 dkimpy==1.1.8 -fastapi==0.143.3 +fastapi==0.149.0 GitPython==3.2.1 grpc-interceptor==0.15.4 grpcio==1.84.0 From f426843d813ce5cf45902c7e12dbd31a130b062c Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Thu, 22 Oct 2026 19:14:46 +0100 Subject: [PATCH 04/13] docs: move the docs into an mdBook manual --- .github/workflows/docs.yml | 40 ++++ .gitignore | 7 +- CONTRIBUTING.md | 20 +- INSTALL.md | 2 +- Makefile | 10 +- README.md | 33 ++-- book.toml | 19 ++ docs/README.md | 52 +++++ docs/SUMMARY.md | 68 +++++++ .../smtp-transport.md} | 25 ++- .../developers/reference-client.md | 8 +- .../relaysms-whitepaper.pdf} | Bin .../manage.md} | 20 +- docs/operating/observability.md | 148 ++++++++++++++ docs/{platforms.md => platforms/manage.md} | 16 +- docs/{grpc.md => reference/grpc-api.md} | 119 ++++------- docs/{rest.md => reference/rest-api.md} | 27 ++- observability/README.md | 185 ------------------ .../nginx-observability.conf.template | 2 +- observability/nginx-uptime-kuma.conf.template | 2 +- observability/signoz/casting.yaml.template | 2 +- publisher/gateway_clients/mcc_mnc.py | 2 +- publisher/tasks/celery_app.py | 2 +- requirements-observability.txt | 2 +- scripts/build-book.sh | 88 +++++++++ scripts/setup-observability.sh | 4 +- template.env | 4 +- tests/scripts/test_build_book.py | 54 +++++ 28 files changed, 602 insertions(+), 359 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 book.toml create mode 100644 docs/README.md create mode 100644 docs/SUMMARY.md rename docs/{smtp.md => configuration/smtp-transport.md} (73%) rename tools/README.md => docs/developers/reference-client.md (93%) rename docs/{RelaySMS - Technical whitepaper.pdf => explanation/relaysms-whitepaper.pdf} (100%) rename docs/{gateway-clients.md => gateway-clients/manage.md} (57%) create mode 100644 docs/operating/observability.md rename docs/{platforms.md => platforms/manage.md} (80%) rename docs/{grpc.md => reference/grpc-api.md} (79%) rename docs/{rest.md => reference/rest-api.md} (80%) delete mode 100644 observability/README.md create mode 100755 scripts/build-book.sh create mode 100644 tests/scripts/test_build_book.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..a8c80b2 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,40 @@ +name: Docs + +# Checks Markdown links and builds the manual. +on: + pull_request: + branches: + - staging + - main + paths: + - "**.md" + - "docs/**" + - "book.toml" + - "scripts/build-book.sh" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + # Local files and anchors only, so an external site being down can't fail a PR. + - name: Check Markdown links + uses: lycheeverse/lychee-action@v2 + with: + args: --offline --include-fragments --no-progress README.md INSTALL.md CONTRIBUTING.md sdk/README.md "docs/**/*.md" + + - name: Build + run: make book + + # Catches links that resolve on GitHub but not in the book, such as to files outside docs/. + - name: Check book links + uses: lycheeverse/lychee-action@v2 + with: + args: --offline --include-fragments --no-progress --exclude-path book/404.html "book/**/*.html" diff --git a/.gitignore b/.gitignore index eca77a7..2ab87d2 100644 --- a/.gitignore +++ b/.gitignore @@ -83,8 +83,11 @@ instance/ # Scrapy stuff: .scrapy -# Sphinx documentation -docs/_build/ +# mdBook output and tools, see scripts/build-book.sh +/book/ +/.mdbook/ +/mermaid.min.js +/mermaid-init.js # PyBuilder target/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cbbeec0..ba0ddfc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,7 +88,7 @@ Each service can also run on its own: | Celery worker and beat | `celery -A publisher.tasks.celery_app:celery_app worker` / `beat` | | CLI | `python -m publisher --help` (or `./publisher.sh`) | -`python -m tools.client` drives the gRPC and REST flows by hand, see [tools/README.md](tools/README.md). +`python -m tools.client` drives the gRPC and REST flows by hand, see [docs/developers/reference-client.md](docs/developers/reference-client.md). ### Make targets @@ -104,6 +104,8 @@ Each service can also run on its own: | `make test` | Run pytest | | `make coverage` | Run pytest and list the lines no test runs | | `make docs` | Regenerate `docs/openapi.json` from the REST app | +| `make book` | Build the manual in `docs/` into `book/` | +| `make book-serve` | Preview the manual on , rebuilding on changes | | `make test-dialects` | Migrate and query SQLite, SQLCipher, Postgres, MySQL and MariaDB (podman) | | `make test-e2e` | Install, update and uninstall in a systemd container (podman) | | `make check` | Run every pre-commit hook on all files | @@ -306,14 +308,14 @@ make build test PYTHON=.venv312/bin/python 1. Add the route to the matching module in `publisher/api/rest/v1/` (or a new module with its own `APIRouter`, registered in `routes.py`). 2. Put request and response models in `schemas.py`. Use `Depends(get_db)` for the session, and `Security(authorize, scopes=[...])` if it needs a login. A new scope goes in `Scope` in `publisher/models/credential.py`. 3. Map domain errors to `ApiError` (`errors.py`) with a message a client can show, and put ids and other detail in its `log=`. -4. Test it in `tests/api/rest/`, describe it in its docstring and run `make docs`. Update [docs/rest.md](docs/rest.md) only for what the reference doesn't show. +4. Test it in `tests/api/rest/`, describe it in its docstring and run `make docs`. Update [docs/reference/rest-api.md](docs/reference/rest-api.md) only for what the reference doesn't show. ### Change the gRPC API 1. Edit `protos/v3/publisher.proto` and run `make protos`. 2. Add a module per RPC in `publisher/api/grpc/v3/` and a thin method on the servicer that calls it. 3. Raise `grpc_interceptor` exceptions such as `InvalidArgument` or `Unauthenticated`; `ErrorInterceptor` turns them into status codes and anything else into `INTERNAL`. -4. Test it in `tests/api/grpc/` and update [docs/grpc.md](docs/grpc.md). +4. Test it in `tests/api/grpc/` and update [docs/reference/grpc-api.md](docs/reference/grpc-api.md). > [!WARNING] > Deployed clients depend on the proto: only add fields. @@ -328,13 +330,17 @@ Add it under `publisher/tasks/`. Keep the explicit `tasks.*` name so queued mess ### Platform adapters -Adapters live in their own repositories and are built with the SDK in `sdk/` ([sdk/README.md](sdk/README.md)). Their CI runs `.github/workflows/adapter.yml` at the SDK tag they pin, so a change to it reaches an adapter only when it moves its pin. Install one locally with `python -m publisher platforms add `; on a server use `./publisher.sh platforms`, which runs as the service user. See [docs/platforms.md](docs/platforms.md). +Adapters live in their own repositories and are built with the SDK in `sdk/` ([sdk/README.md](sdk/README.md)). Their CI runs `.github/workflows/adapter.yml` at the SDK tag they pin, so a change to it reaches an adapter only when it moves its pin. Install one locally with `python -m publisher platforms add `; on a server use `./publisher.sh platforms`, which runs as the service user. See [docs/platforms/manage.md](docs/platforms/manage.md). ## Documentation -- [docs/rest.md](docs/rest.md) (with the generated [docs/openapi.json](docs/openapi.json)), [docs/grpc.md](docs/grpc.md) and [docs/smtp.md](docs/smtp.md) describe each interface. Update them in the same PR as the behaviour. -- The README covers installation, configuration and operations. Keep it in step with `template.env` and the CLI. -- Write plainly and briefly. Use code blocks for commands and tables for options. +The manual in `docs/` is an [mdBook](https://rust-lang.github.io/mdBook/) that reads the same on GitHub. Preview it with `make book-serve`. + +- Update the pages a change affects in the same PR. +- List a new page in `docs/SUMMARY.md`, or the book leaves it out. +- Link pages in `docs/` relatively, and files outside it by their full GitHub URL, since the book only has `docs/`. +- Draw diagrams as `mermaid` code blocks. +- Write plainly and briefly, one paragraph per line, with sentence-case headings, code blocks for commands and tables for options. ## When CI fails diff --git a/INSTALL.md b/INSTALL.md index 3bb233d..e2d4424 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -165,7 +165,7 @@ The [README](README.md#manage-the-services) lists the `manage.sh` commands. Run ./publisher.sh shell # a shell as the service user with .env loaded ``` -[Platform adapters](docs/platforms.md), [gateway clients](docs/gateway-clients.md) and [credentials](README.md#credentials) each have their own guide. Observability is optional; see [observability/README.md](observability/README.md). +[Platform adapters](docs/platforms/manage.md), [gateway clients](docs/gateway-clients/manage.md) and [credentials](README.md#credentials) each have their own guide. Observability is optional; see [docs/operating/observability.md](docs/operating/observability.md). ## Configuration diff --git a/Makefile b/Makefile index 20c6016..726df6a 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ CREDS_USER ?= dev SPECS_DIR := lib_relaysms_payload_specs SPECS_LIB := $(SPECS_DIR)/target/release/librelaysms_spec_payload.so -.PHONY: setup build protos specs migrate creds run test coverage test-sdk test-dialects test-e2e docs check clean +.PHONY: setup build protos specs migrate creds run test coverage test-sdk test-dialects test-e2e docs book book-serve check clean ## Set up a development checkout. setup: venv/bin/python .env @@ -76,6 +76,14 @@ test-e2e: docs: $(PYTHON) -m publisher.api.rest.openapi +## Build the manual in docs/ into book/. +book: + ./scripts/build-book.sh + +## Serve the manual on http://localhost:3000, rebuilding on changes. +book-serve: + ./scripts/build-book.sh --serve + check: $(PYTHON) -m pre_commit run --all-files diff --git a/README.md b/README.md index e96621c..ebd1ac7 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The installer sets up everything else, including the system packages and Rust. | `--site-name DOMAIN` | Puts nginx with a Let's Encrypt certificate in front; `--skip-nginx` skips it | | `--setup-db mysql\|postgres` | Installs the database server and writes its details to `.env` ([details](INSTALL.md#database)) | | `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker, for heavier load ([details](INSTALL.md#celery-worker-and-beat)) | - | `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](observability/README.md)) | + | `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](docs/operating/observability.md)) | | `--install-dir PATH`, `--instance-name NAME` | Installs elsewhere, or a second copy on the same host ([details](INSTALL.md#running-multiple-instances)) | `install.sh --help` lists every flag. @@ -165,14 +165,14 @@ Settings live in `.env`. `template.env` lists every setting with its default and | Settings | Documented in | | --- | --- | | Server, database, encryption, Celery, adapters | [INSTALL.md](INSTALL.md#configuration) | -| REST API login and sessions (`AUTH_*`) | [REST API](docs/rest.md#authentication) | -| SMTP transport (`SMTP_*`) | [SMTP Transport](docs/smtp.md#configuration) | -| Tracing, metrics and logs (`OTEL_*`) | [Observability](observability/README.md) | +| REST API login and sessions (`AUTH_*`) | [REST API](docs/reference/rest-api.md#authentication) | +| SMTP transport (`SMTP_*`) | [SMTP transport](docs/configuration/smtp-transport.md#configuration) | +| Tracing, metrics and logs (`OTEL_*`) | [Observability](docs/operating/observability.md) | | Offline publishing (`OFFLINE_PUBLISH_*`) | [Below](#offline-publishing) | ### Offline Publishing -Offline payloads are tagged with the protocol they arrived on: `https` for [REST `/publications`](docs/rest.md#publishing) and `smtp` for the [SMTP transport](docs/smtp.md). +Offline payloads are tagged with the protocol they arrived on: `https` for [REST `/publications`](docs/reference/rest-api.md#publishing) and `smtp` for the [SMTP transport](docs/configuration/smtp-transport.md). ```bash OFFLINE_PUBLISH_ALLOWED_PROTOCOLS=smtp # comma-separated; empty allows every protocol @@ -197,15 +197,15 @@ Each platform is served by an adapter installed from its own repository. `GET /v - [Bluesky](https://github.com/smswithoutborders/bluesky-oauth2-adapter) - [Mastodon](https://github.com/smswithoutborders/mastodon-oauth2-adapter) -[docs/platforms.md](docs/platforms.md) covers adding, updating, disabling and removing adapters. Each adapter's README covers its own settings. +[docs/platforms/manage.md](docs/platforms/manage.md) covers adding, updating, disabling and removing adapters. Each adapter's README covers its own settings. ## Gateway clients -Gateway clients are the phone numbers that relay SMS to this server. `GET /v1/gateway-clients` lists the enabled ones, and [docs/gateway-clients.md](docs/gateway-clients.md) covers managing them. +Gateway clients are the phone numbers that relay SMS to this server. `GET /v1/gateway-clients` lists the enabled ones, and [docs/gateway-clients/manage.md](docs/gateway-clients/manage.md) covers managing them. ## Credentials -Credentials log in to the REST API, each with its own scopes. One holding every scope is an administrator. They can also be managed over the [REST API](docs/rest.md#managing-credentials). +Credentials log in to the REST API, each with its own scopes. One holding every scope is an administrator. They can also be managed over the [REST API](docs/reference/rest-api.md#managing-credentials). ```bash ./publisher.sh creds scopes # list scopes @@ -222,14 +222,15 @@ Credentials log in to the REST API, each with its own scopes. One holding every ## Documentation -- [Installation Guide](INSTALL.md): installing by hand and every setting -- [REST API](docs/rest.md): publishing, authentication and the [OpenAPI reference](docs/openapi.json) -- [gRPC API](docs/grpc.md): linking accounts and syncing keys -- [SMTP Transport](docs/smtp.md): publishing payloads sent by email -- [Platform Adapters](docs/platforms.md): installing and managing adapters -- [Gateway Clients](docs/gateway-clients.md): managing gateway clients -- [Observability](observability/README.md): tracing, metrics, logs and uptime monitoring -- [Reference Client](tools/README.md): exercising the gRPC and REST flows by hand +- [Manual](docs/README.md): installing, configuring and running the Publisher +- [Installation guide](INSTALL.md): installing by hand and every setting +- [REST API](docs/reference/rest-api.md): publishing, authentication and the [OpenAPI reference](docs/openapi.json) +- [gRPC API](docs/reference/grpc-api.md): linking accounts and syncing keys +- [SMTP transport](docs/configuration/smtp-transport.md): publishing payloads sent by email +- [Platform adapters](docs/platforms/manage.md): installing and managing adapters +- [Gateway clients](docs/gateway-clients/manage.md): managing gateway clients +- [Observability](docs/operating/observability.md): tracing, metrics, logs and uptime monitoring +- [Reference client](docs/developers/reference-client.md): exercising the gRPC and REST flows by hand - [Contributing](CONTRIBUTING.md): setup, workflow, conventions and testing ## License diff --git a/book.toml b/book.toml new file mode 100644 index 0000000..5806840 --- /dev/null +++ b/book.toml @@ -0,0 +1,19 @@ +[book] +title = "RelaySMS Publisher" +description = "Install, configure and run the RelaySMS Publisher." +language = "en" +src = "docs" + +[build] +create-missing = false +# Without the index preprocessor, README.md pages stay README.html, so links to them work in the book as on GitHub. +use-default-preprocessors = false + +[preprocessor.mermaid] +command = "mdbook-mermaid" + +[output.html] +git-repository-url = "https://github.com/smswithoutborders/RelaySMS-Publisher" +edit-url-template = "https://github.com/smswithoutborders/RelaySMS-Publisher/edit/main/{path}" +additional-js = ["mermaid.min.js", "mermaid-init.js"] +no-section-label = true diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..1cceecd --- /dev/null +++ b/docs/README.md @@ -0,0 +1,52 @@ +# RelaySMS Publisher + +The Publisher posts to online platforms (Gmail, Telegram, Mastodon and more) from content people send by SMS, so they can publish without an internet connection. + +## How it works + +```mermaid +flowchart TB + app["RelaySMS app"] + gateway["Gateway client"] + mailbox["Mailbox"] + platform["Gmail, Telegram, ..."] + + subgraph publisher["Publisher"] + grpc["gRPC API"] + rest["REST API"] + smtp["Email listener"] + worker["Celery worker"] + adapter["Platform adapter"] + end + + app -- "link an account, online" --> grpc + app -- "encrypted SMS" --> gateway + gateway -- "HTTPS" --> rest + gateway -- "email" --> mailbox + smtp -- "IMAP" --> mailbox + rest --> worker + smtp --> worker + grpc --> adapter + worker --> adapter + adapter --> platform +``` + +1. While online, a user links an account through the gRPC API. The Publisher stores its token encrypted. +2. Offline, the app encrypts the content and sends it by SMS to a gateway client, a phone number that relays SMS. +3. The gateway client posts the payload to the REST API, or emails it to a mailbox the Publisher reads. +4. A Celery worker decrypts the payload and publishes it through the platform's adapter. + +## What you need + +- A server running Ubuntu 24.04 or Debian 13, with a domain name pointing at it +- Developer credentials from each platform you offer, such as a Google Cloud OAuth client for Gmail +- At least one gateway client: a phone number whose SMS reach the REST API or a mailbox + +## Where to start + +| To | Read | +| --- | --- | +| Offer a platform | [Managing adapters](platforms/manage.md) | +| Relay SMS to the Publisher | [Managing gateway clients](gateway-clients/manage.md) | +| Build a client | [gRPC API](reference/grpc-api.md) and [REST API](reference/rest-api.md) | +| Understand the protocol | [RelaySMS whitepaper](explanation/relaysms-whitepaper.pdf) | diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md new file mode 100644 index 0000000..be849a1 --- /dev/null +++ b/docs/SUMMARY.md @@ -0,0 +1,68 @@ +# Summary + +[Introduction](README.md) + +# Getting started + +- [Quick start]() + +# Installation + +- [Requirements]() +- [Install on a server]() +- [Install with Docker]() +- [Install by hand]() +- [Database]() +- [Celery broker]() +- [Reverse proxy and TLS]() +- [Multiple instances]() + +# Configuration + +- [Settings reference]() +- [Encryption keys]() +- [Offline publishing]() +- [SMTP transport](configuration/smtp-transport.md) + +# Platform adapters + +- [Managing adapters](platforms/manage.md) +- [Gmail]() +- [Mastodon]() +- [Bluesky]() +- [Telegram]() +- [RelaySMS Mail]() + +# Gateway clients + +- [Managing gateway clients](gateway-clients/manage.md) +- [Connecting a relay]() + +# Operating + +- [Services]() +- [Credentials and access]() +- [Upgrading]() +- [Backup and restore]() +- [Observability](operating/observability.md) +- [Audit log]() +- [Security hardening]() +- [Troubleshooting]() + +# Reference + +- [Command-line tools]() +- [REST API](reference/rest-api.md) +- [gRPC API](reference/grpc-api.md) +- [Glossary]() + +# Explanation + +- [Architecture]() +- [Security model]() +- [Data and retention]() + +# Developers + +- [Writing an adapter]() +- [Reference client](developers/reference-client.md) diff --git a/docs/smtp.md b/docs/configuration/smtp-transport.md similarity index 73% rename from docs/smtp.md rename to docs/configuration/smtp-transport.md index 98dc2c7..f7319ed 100644 --- a/docs/smtp.md +++ b/docs/configuration/smtp-transport.md @@ -1,8 +1,8 @@ -# SMTP Transport +# SMTP transport -Publishes RelaySMS payloads received by email. Incoming mails are picked up by polling a mailbox over IMAP (`publisher/smtp/listener.py`), authenticated, and queued for publication. +Publishes RelaySMS payloads that gateway clients send by email. The listener watches a mailbox over IMAP, checks each sender, and queues the payload for publication. -## Message Format +## Message format The body of each email must be a JSON object: @@ -20,13 +20,13 @@ The body of each email must be a JSON object: | dialing_code | string | Optional. Sender's country calling code, for publication stats | | text | string | Base64-encoded serialized payload | -## How It Works +## How it works 1. Waits for unseen mail in `SMTP_IMAP_MAIL_FOLDER` with IMAP IDLE. List several folders, such as `INBOX,Spam`, to catch mail a provider misfiled. 2. Rejects each email unless the sender is allowlisted and authenticated (see [Security](#security)). 3. Parses the body as JSON and validates the payload. 4. Queues the payload for publication. -5. Deletes the handled emails, in one call per folder. +5. Deletes each handled email. An email that hits an unexpected error stays in the folder and is retried on the next cycle. ## Security @@ -48,19 +48,16 @@ Two checks must both pass before a message is queued: | `SMTP_IMAP_PORT` | `993` | IMAP port. | | `SMTP_IMAP_USERNAME` | - | Required when enabled. | | `SMTP_IMAP_PASSWORD` | - | Required when enabled. | -| `SMTP_IMAP_MAIL_FOLDER` | `INBOX` | Comma-separated folders to poll. | -| `SMTP_TLS_CLIENT_CERTIFICATE` / `SMTP_TLS_CLIENT_KEY` | - | Optional mTLS client cert, only if the provider requires one. | +| `SMTP_IMAP_MAIL_FOLDER` | `INBOX` | Comma-separated folders to watch. | +| `SMTP_TLS_CLIENT_CERTIFICATE` / `SMTP_TLS_CLIENT_KEY` | - | Optional mTLS client certificate and key, if the provider requires one. Set both or neither. | | `SMTP_ALLOWED_SENDERS` | - | Comma-separated allowlist of addresses and/or domains. | -| `SMTP_TRUSTED_AUTHSERV_ID` | - | `authserv-id` whose `Authentication-Results` verdicts are trusted. | +| `SMTP_TRUSTED_AUTHSERV_ID` | - | `authserv-id` whose `Authentication-Results` verdicts are trusted. Required when enabled. | | `SMTP_REQUIRE_DKIM` | `true` | Require `dkim=pass`. | | `SMTP_REQUIRE_SPF` | `true` | Require `spf=pass`. | | `SMTP_VERIFY_DKIM_INDEPENDENTLY` | `false` | Also re-verify the DKIM signature via DNS. | -| `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` | - | Comma-separated allowlist of protocols allowed to publish offline payloads. Messages queued by this listener are tagged `smtp`; if set and `smtp` isn't listed, offline payloads from email are discarded. | -## Running +The listener tags what it queues with protocol `smtp`. Whether offline payloads from email publish depends on [offline publishing](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#offline-publishing). -```sh -python3 -m publisher.smtp -``` +## Running -`scripts/run.sh` and the `relaysms-publisher-smtp` unit start it too; it exits at once while `SMTP_TRANSPORT_ENABLED` is off. +Set `SMTP_TRANSPORT_ENABLED=true`, then run `sudo ./manage.sh restart`. The `relaysms-publisher-smtp` service runs the listener and exits at once while the setting is off. diff --git a/tools/README.md b/docs/developers/reference-client.md similarity index 93% rename from tools/README.md rename to docs/developers/reference-client.md index cb7d1b6..483899c 100644 --- a/tools/README.md +++ b/docs/developers/reference-client.md @@ -1,13 +1,13 @@ # Reference Client -`tools/client.py` drives the gRPC and REST flows by hand, the way a phone app would. It keeps tokens and key pools in local state between runs. +`tools/client.py` drives the gRPC and REST flows by hand, the way a phone app would. It keeps tokens and key pools in `tools/db.json` between runs. ```bash pip install -r requirements.txt python -m tools.client [ARGS] ``` -## Connection Options +## Connection options | Option | Default | Does | | --- | --- | --- | @@ -15,10 +15,10 @@ python -m tools.client [ARGS] | `--tls` | off | Use TLS for gRPC | | `--rest-api` | `http://localhost:16000` | REST API base URL | | `--platform`, `-p` | | Platform name, such as `gmail` or `telegram` | -| `--phone-number` | | Phone number, for PNBA | +| `--phone-number` | | Phone number, for phone number-based authentication (PNBA, e.g. Telegram) | | `--request-identifier` | | Optional request identifier | -## Linking Accounts +## Linking accounts | Command | Does | | --- | --- | diff --git a/docs/RelaySMS - Technical whitepaper.pdf b/docs/explanation/relaysms-whitepaper.pdf similarity index 100% rename from docs/RelaySMS - Technical whitepaper.pdf rename to docs/explanation/relaysms-whitepaper.pdf diff --git a/docs/gateway-clients.md b/docs/gateway-clients/manage.md similarity index 57% rename from docs/gateway-clients.md rename to docs/gateway-clients/manage.md index db3a55c..a135597 100644 --- a/docs/gateway-clients.md +++ b/docs/gateway-clients/manage.md @@ -1,25 +1,25 @@ -# Gateway Clients +# Managing gateway clients -Gateway clients are the phone numbers that relay SMS to this server. They're stored in the `gateway_clients` table and listed publicly at `GET /v1/gateway-clients`; credentials with `gc:read` and `gc:write` manage them over the [REST API](rest.md#managing-gateway-clients). Every change is in the audit log. +Gateway clients are the phone numbers that relay SMS to this server. The enabled ones are listed publicly at `GET /v1/gateway-clients`. Credentials with `gc:read` and `gc:write` can manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients). Every change is recorded in the [audit log](../reference/rest-api.md#audit-events). ## Commands ```bash ./publisher.sh gateway-clients suggest # details and PLMN candidates; saves nothing -./publisher.sh gateway-clients create --msisdn --protocols [--country] [--operator] [--operator-code] -./publisher.sh gateway-clients list [--msisdn] [--country] [--operator] -./publisher.sh gateway-clients update [--country] [--operator] [--operator-code] [--protocols] +./publisher.sh gateway-clients create --msisdn --protocols [--country ] [--operator ] [--operator-code ] +./publisher.sh gateway-clients list [--msisdn ] [--country ] [--operator ] +./publisher.sh gateway-clients update [--country ] [--operator ] [--operator-code ] [--protocols ] ./publisher.sh gateway-clients disable # hide it from the public list ./publisher.sh gateway-clients enable ./publisher.sh gateway-clients delete ./publisher.sh gateway-clients countries ./publisher.sh gateway-clients operators --country -./publisher.sh gateway-clients mcc-mnc list [--country-code] [--network] [--iso] # search the PLMN table +./publisher.sh gateway-clients mcc-mnc list [--country-code ] [--network ] [--iso ] # search the PLMN table ``` -`create` fills in what you leave out from `suggest`, the PLMN code only when it's unambiguous. Otherwise pick a candidate and pass it with `--operator-code`. +`create` fills in any of `--country`, `--operator` and `--operator-code` you leave out with what `suggest` finds. It fills the PLMN code (MCC+MNC) only when every candidate shares it; otherwise run `suggest`, pick a candidate and pass it with `--operator-code`. -## How Suggest Works +## How suggest works `phonenumbers` gives the number's country, region and carrier name. The carrier is matched against that region's operators in `publisher/gateway_clients/mcc_mnc_table.json`, a vendored snapshot of [musalbas/mcc-mnc-table](https://github.com/musalbas/mcc-mnc-table). A territory without rows of its own, like Guernsey, uses its calling code's main region. @@ -34,6 +34,6 @@ Names are compared word by word, ignoring case, accents, company-type words ("Te > [!TIP] > A PLMN missing from the snapshot can still be entered with `--operator-code`. -## Upgrading From the JSON Registry +## Upgrading from the JSON registry -Migration 019 imports `data/gateway_clients/registry.json` (or `GATEWAY_CLIENTS_REGISTRY_FILE`, if `.env` still sets it). MCC/MNC overrides are no longer used; the migration logs any it finds instead. Both files can then be deleted. +Migration 019 imports `data/gateway_clients/registry.json` (or `GATEWAY_CLIENTS_REGISTRY_FILE`, if `.env` sets it). It logs any entries in `mcc_mnc_overrides.json` without importing them. Both files can then be deleted. diff --git a/docs/operating/observability.md b/docs/operating/observability.md new file mode 100644 index 0000000..ac7ba54 --- /dev/null +++ b/docs/operating/observability.md @@ -0,0 +1,148 @@ +# Observability + +Self-hosted tracing, metrics and log export (SigNoz) and uptime monitoring (Uptime Kuma) for the Publisher's five services. Both are optional; tracing stays off while `OTEL_EXPORTER_OTLP_ENDPOINT` is blank. + +## Setup + +```bash +sudo ./install.sh --setup-observability # during install +sudo ./scripts/setup-observability.sh # on an existing install +``` + +| `install.sh` | `setup-observability.sh` | Does | +| --- | --- | --- | +| `--observability-site-name DOMAIN` | `--site-name DOMAIN` | Puts nginx and a Let's Encrypt certificate in front of SigNoz | +| `--observability-kuma-site-name DOMAIN` | `--kuma-site-name DOMAIN` | The same for Uptime Kuma, on a different domain | +| - | `--signoz-email`, `--signoz-password` | SigNoz admin account | +| - | `--signoz-smtp-host`, `-port`, `-username`, `-password`, `-from` | SMTP relay for SigNoz's email alerts | +| - | `--skip-tracing` | Leaves tracing off | + +`scripts/setup-observability.sh --help` lists every flag. The script installs Docker if it's missing. + +## SigNoz + +**Automated**: writes `observability/signoz/casting.yaml` from its template (Postgres credentials, ports, SMTP), binds the UI and OTLP ports to `127.0.0.1`, points the collector's opamp at SigNoz so the OTLP receiver opens, starts SigNoz and creates the admin account. + +**Manual** (needs Docker Engine with the Compose plugin): + +```bash +curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_INSTALL_DIR=/usr/local/bin bash + +sed \ + -e "s/__POSTGRES_DB__/signoz/" -e "s/__POSTGRES_USER__/signoz/" \ + -e "s/__POSTGRES_PASSWORD__/$(openssl rand -hex 24)/" \ + -e "s/__SIGNOZ_PORT__/8090/" -e "s/__OTLP_GRPC_PORT__/4317/" -e "s/__OTLP_HTTP_PORT__/4318/" \ + -e "s/__SMTP_SMARTHOST__//" -e "s/__SMTP_USERNAME__//" -e "s/__SMTP_PASSWORD__//" -e "s/__SMTP_FROM__//" \ + -e "s|__SIGNOZ_EXTERNAL_URL__|http://localhost:8090|" \ + observability/signoz/casting.yaml.template > observability/signoz/casting.yaml + +foundryctl cast -f observability/signoz/casting.yaml +docker compose \ + -f pours/deployment/compose.yaml \ + -f observability/signoz/docker-compose.override.yml \ + up -d + +until curl -fs http://127.0.0.1:8090/api/v1/version | grep -q setupCompleted; do sleep 2; done +curl -X POST http://127.0.0.1:8090/api/v1/register \ + -H "Content-Type: application/json" \ + -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@relaysms.local","password":"<12+ chars, upper, lower, number, symbol>"}' +``` + +The template stores SigNoz's metadata in Postgres (`metastore.kind: postgres`); set `sqlite` for a single node. Set retention in the UI, under Settings > Workspace > Retention Controls. + +> [!IMPORTANT] +> `casting.yaml` is generated only once. To change a running install, edit `observability/signoz/casting.yaml` and run `foundryctl cast -f observability/signoz/casting.yaml` again. It recreates only the `signoz` container; Postgres and ClickHouse keep their data. + +## Uptime Kuma + +**Automated**: starts the container on `127.0.0.1:3001`, or the port you choose, which it saves to `observability/uptime-kuma/.env`. + +**Manual**: `docker compose -f observability/uptime-kuma/docker-compose.yml up -d` + +Open it through its own domain ([below](#reverse-proxy)) or an SSH tunnel (`ssh -L 3001:127.0.0.1:3001 @`, then ). Create the admin account on the first visit, then add these monitors: + +- **REST**: Monitor Type "HTTP(s)", URL `https:///health`. +- **gRPC**: Monitor Type "GRPC(S) - Keyword", with: + - gRPC URL: `:443` + - Enable TLS: on + - Service Name: `grpc.health.v1.Health` + - Method: `check` + - gRPC Body: `{"service":"publisher.v3.Publisher"}` + - Keyword: `SERVING` + - Protobuf: + + ```proto + syntax = "proto3"; + package grpc.health.v1; + + message HealthCheckRequest { + string service = 1; + } + + message HealthCheckResponse { + enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; + SERVICE_UNKNOWN = 3; + } + ServingStatus status = 1; + } + + service Health { + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); + } + ``` + +- **Worker**: Monitor Type "Push", interval just over 60s. Set `UPTIME_KUMA_WORKER_PUSH_URL` in `.env` to the push URL Kuma shows, then run `sudo ./manage.sh restart`. Beat schedules this heartbeat, so the monitor also goes down when Beat stops. +- **SMTP listener**: as for the worker, with an interval just over 20s and `UPTIME_KUMA_SMTP_PUSH_URL`. + +## Reverse proxy + +**Automated**: `--site-name` and `--kuma-site-name` on `setup-observability.sh` write and enable the nginx sites and run Certbot. + +**Manual**: + +```bash +sed -e "s/__SERVER_NAME__//g" \ + observability/nginx-observability.conf.template \ + > /etc/nginx/sites-available/.conf +ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ + +sed -e "s/__SERVER_NAME__//g" \ + observability/nginx-uptime-kuma.conf.template \ + > /etc/nginx/sites-available/.conf +ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ + +nginx -t && systemctl reload nginx +certbot --nginx -d +certbot --nginx -d +``` + +## Turning tracing on and off + +`setup-observability.sh` installs `requirements-observability.txt` and sets the endpoint unless given `--skip-tracing`. To do it by hand, from the install directory: + +```bash +sudo venv/bin/pip install --quiet -r requirements-observability.txt +sudo sed -i 's|^OTEL_EXPORTER_OTLP_ENDPOINT=.*|OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317|' .env +sudo ./manage.sh restart +``` + +Blank the value and restart to turn tracing off. + +## Alerts + +In the SigNoz UI, go to Alerts > New Alert > Logs, paste a filter below into the search bar, set the threshold to count > 0 over a rolling window, and attach a notification channel. Keep the parentheses: `AND` binds tighter than `OR`. + +- **Publish failures**, adapter errors included: + + ``` + service.name = 'relaysms-publisher-worker' AND (body CONTAINS 'Failed to process payload' OR body CONTAINS 'Failed to publish message' OR body CONTAINS 'An unexpected error occurred during task processing') + ``` + +- **Key management failures**: + + ``` + body CONTAINS 'Failed to generate server identity keys' OR body CONTAINS 'Token hash missing' + ``` diff --git a/docs/platforms.md b/docs/platforms/manage.md similarity index 80% rename from docs/platforms.md rename to docs/platforms/manage.md index b04042e..2e11eed 100644 --- a/docs/platforms.md +++ b/docs/platforms/manage.md @@ -1,6 +1,6 @@ -# Platform Adapters +# Managing adapters -Each platform (Gmail, X, Telegram, ...) is served by an adapter installed from its own GitHub repository. Adapters are registered in the `platform_adapters` table; their files live under `data/platforms/` (`PLATFORMS_ADAPTERS_*` in `.env`). +Each platform (Gmail, Telegram, Mastodon, ...) is served by an adapter installed from its own GitHub repository. Its files live under `data/platforms/` (see [Files](#files)). > [!IMPORTANT] > Run these commands with `./publisher.sh` from the install directory, not `python3 -m publisher`. It runs as the service user, so the services can still read the adapter files. @@ -21,7 +21,7 @@ Each platform (Gmail, X, Telegram, ...) is served by an adapter installed from i ./publisher.sh platforms exec -- [ARGS] # run a command the adapter installs ``` -Commands taking a name also take `--proto-id` and `--cat-id`, for when a name matches more than one adapter. Administrators can do the same over the [REST API](rest.md#managing-platform-adapters), installing only from orgs in `PLATFORMS_GITHUB_ORGS`. +Commands taking a name also take `--proto-id` and `--cat-id`, for when a name matches more than one adapter. Administrators can do the same over the [REST API](../reference/rest-api.md#managing-platform-adapters), but only from GitHub orgs listed in `PLATFORMS_GITHUB_ORGS`. ## Versions @@ -30,7 +30,7 @@ Adapters are installed and updated by version tag, such as `v1.2.0`, so a reposi An update builds the new version and its virtualenv beside the running one, then switches over, so a failed update leaves the running version in place. > [!NOTE] -> An update is refused if the installed tag now points to a different commit. Release tags shouldn't move. +> An update is refused if the installed tag now points to a different commit. Publish a new tag instead of moving one. ## Removing @@ -58,13 +58,13 @@ An adapter reads its settings, such as API keys, from `credentials.json` in its ./publisher.sh platforms config set rmail --unset alias_prefix ``` -`config set` changes only the keys given. `setup` is for adapters whose settings come from the platform, such as an app it registers or a file its console downloads; it replaces the whole file. `--input KEY=@PATH` reads a file: +`config set` changes only the keys given. `setup` is for adapters whose settings come from the platform (e.g. a registered app for Mastodon, a downloaded client file for Gmail) and replaces all of them. `--input KEY=@PATH` reads a file: ```bash ./publisher.sh platforms setup gmail --input client_file=@client_secret.json ``` -## An Adapter's Own Commands +## An adapter's own commands An adapter may install admin commands. `exec` runs one from the adapter's virtualenv with its config and state directories; put `--` before it: @@ -72,6 +72,6 @@ An adapter may install admin commands. `exec` runs one from the adapter's virtua ./publisher.sh platforms exec -- --help ``` -## Writing an Adapter +## Writing an adapter -Build it with the [adapter SDK](../sdk/README.md), then tag a release and `add` it. +Build it with the [adapter SDK](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk), then tag a release and `add` it. diff --git a/docs/grpc.md b/docs/reference/grpc-api.md similarity index 79% rename from docs/grpc.md rename to docs/reference/grpc-api.md index 9d5fa9c..83d4293 100644 --- a/docs/grpc.md +++ b/docs/reference/grpc-api.md @@ -1,80 +1,41 @@ -# Publisher gRPC Documentation +# gRPC API -## Table of Contents +The gRPC API links and unlinks platform accounts and syncs ephemeral key pools. The RelaySMS app calls it while online. -- [Download Protocol Buffer Files](#download-protocol-buffer-files) -- [Prerequisites](#prerequisites) -- [Version 3 API](#version-3-api) - - [v3: Get OAuth2 Authorization URL](#v3-get-oauth2-authorization-url) - - [v3: Exchange OAuth2 Code and Store Token](#v3-exchange-oauth2-code-and-store-token) - - [v3: Revoke and Delete OAuth2 Token](#v3-revoke-and-delete-oauth2-token) - - [v3: Get PNBA Code](#v3-get-pnba-code) - - [v3: Exchange PNBA Code and Store Token](#v3-exchange-pnba-code-and-store-token) - - [v3: Revoke and Delete PNBA Token](#v3-revoke-and-delete-pnba-token) - - [v3: Sync Keys](#v3-sync-keys) - -## Download Protocol Buffer Files - -### Version 3 +## Protocol buffer file ```bash -curl -O -L https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/staging/protos/v3/publisher.proto +curl -O -L https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/protos/v3/publisher.proto ``` -**Package:** `publisher.v3` -**Service:** `Publisher` - -## Prerequisites +Package `publisher.v3`, service `Publisher`. -### Install Dependencies - -```bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -``` - -For other languages, see [Supported languages](https://grpc.io/docs/languages/). - -### Compile gRPC - -```bash -python -m grpc_tools.protoc -I protos/v3 --python_out=. --grpc_python_out=. protos/v3/publisher.proto -``` +## Generate a client -### Starting the Server +Generate stubs with `protoc` for your [language](https://grpc.io/docs/languages/). For Python: ```bash -GRPC_PORT= \ -GRPC_HOST= \ -python3 -m publisher.api.grpc +pip install grpcio-tools +python -m grpc_tools.protoc -I . --python_out=. --grpc_python_out=. publisher.proto ``` ---- +## Request headers -## Version 3 API - -**Package:** `publisher.v3` -**Service:** `Publisher` - -### v3 Request Headers - -All v3 RPC calls require the following headers: +Every RPC needs these headers. Headers ending in `-bin` carry raw binary. | Header | Type | Required | Description | | :--- | :--- | :--- | :--- | -| `x-payload-bin` | bytes | Yes | Encrypted request payload ciphertext (binary) | -| `x-public-key-bin` | bytes | Yes | Client ephemeral public key ($eC_{pk}$, 32 bytes, binary) | -| `x-key-id` | string | Yes | Server static key ID used for encrypting the payload | -| `x-nonce-bin` | bytes | Yes | Request nonce (16 bytes, binary) | -| `x-timestamp` | string | Yes | Unix timestamp in seconds | +| `x-payload-bin` | bytes | Yes | Encrypted request payload ciphertext | +| `x-public-key-bin` | bytes | Yes | Client ephemeral public key ($eC_{pk}$, 32 bytes) | +| `x-key-id` | string | Yes | ID of the server identity key used to encrypt the payload | +| `x-nonce-bin` | bytes | Yes | Request nonce (16 bytes) | +| `x-timestamp` | string | Yes | Unix timestamp in seconds, within `NONCE_TTL_SECONDS` of the server's clock | -> [!NOTE] -> Headers ending in `-bin` are treated as binary metadata and expect raw binary data. +The examples use `-plaintext` against a local server; drop it for a server behind TLS on port 443. ---- +## Methods -### v3: Get OAuth2 Authorization URL +### Get OAuth2 authorization URL Generates an OAuth2 authorization URL for initiating the OAuth2 flow. @@ -114,13 +75,11 @@ grpcurl -plaintext \ "platform": "gmail", "autogenerate_code_verifier": true }' \ - -proto protos/v3/publisher.proto \ + -proto publisher.proto \ : publisher.v3.Publisher/GetOAuth2AuthorizationUrl ``` ---- - -### v3: Exchange OAuth2 Code and Store Token +### Exchange OAuth2 code and store token Exchanges an OAuth2 authorization code for tokens and stores them. Returns encrypted token data and a set of server ephemeral keys. @@ -163,7 +122,7 @@ grpcurl -plaintext \ -H 'x-nonce-bin: ' \ -H 'x-timestamp: ' \ -d @ \ - -proto protos/v3/publisher.proto \ + -proto publisher.proto \ : publisher.v3.Publisher/ExchangeOAuth2CodeAndStore <: publisher.v3.Publisher/RevokeOAuth2Token ``` ---- - -### v3: Get PNBA Code +### Get PNBA code Requests a one-time passcode (OTP) for Phone Number-Based Authentication. @@ -250,13 +205,11 @@ grpcurl -plaintext \ "platform": "telegram", "phone_number": "+1234567890" }' \ - -proto protos/v3/publisher.proto \ + -proto publisher.proto \ : publisher.v3.Publisher/GetPNBACode ``` ---- - -### v3: Exchange PNBA Code and Store Token +### Exchange PNBA code and store token Exchanges a one-time passcode (OTP) for a token, stores it, and returns encrypted token data. @@ -295,7 +248,7 @@ grpcurl -plaintext \ -H 'x-nonce-bin: ' \ -H 'x-timestamp: ' \ -d @ \ - -proto protos/v3/publisher.proto \ + -proto publisher.proto \ : publisher.v3.Publisher/ExchangePNBACodeAndStore <: publisher.v3.Publisher/RevokePNBAToken ``` ---- +### Sync keys -### v3: Sync Keys - -Syncs the client and server ephemeral key pools for a token. It rotates the client's key pool with a new set of 256 client ephemeral public keys, and the server generates a new pool of 256 server ephemeral keys. +Replaces a token's key pools: the client sends 256 new ephemeral public keys and the server returns 256 of its own. **Request:** `SyncKeysRequest` @@ -379,7 +328,7 @@ grpcurl -plaintext \ -H 'x-nonce-bin: ' \ -H 'x-timestamp: ' \ -d @ \ - -proto protos/v3/publisher.proto \ + -proto publisher.proto \ : publisher.v3.Publisher/SyncKeys < [!NOTE] > Every RPC returns a standard [gRPC status code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html); `0 OK` is success. diff --git a/docs/rest.md b/docs/reference/rest-api.md similarity index 80% rename from docs/rest.md rename to docs/reference/rest-api.md index de725d1..2937ed7 100644 --- a/docs/rest.md +++ b/docs/reference/rest-api.md @@ -1,8 +1,8 @@ -# Publisher REST API +# REST API The REST API serves platform, gateway client and server key metadata, accepts encrypted payloads for publishing, and lets credentials read stats, manage credentials and read the audit log. -The endpoint reference is [openapi.json](openapi.json), generated from the app. Open it in any OpenAPI viewer, or set `API_DOCS_ENABLED=true` on a dev server and browse `/docs`. After changing an endpoint, run `make docs`; a test fails while the file is stale. +The endpoint reference is [openapi.json](../openapi.json), generated from the app. Open it in any OpenAPI viewer, or set `API_DOCS_ENABLED=true` on a development server and browse `/docs`. This page covers what the reference doesn't show well. @@ -16,33 +16,30 @@ https:///v1 ## Authentication -Credentials and their scopes are managed with [`./publisher.sh creds`](../README.md#credentials) or the `/v1/creds` endpoints. +Credentials and their scopes are managed with [`./publisher.sh creds`](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#credentials) or the `/v1/creds` endpoints. -* **Web session:** `POST /v1/auth/login`, then send the cookie with every request (`credentials: "include"` in `fetch`). Writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from this API's origin or `AUTH_WEB_ORIGINS`. Sessions end after 30 minutes idle or 12 hours. +* **Web session:** `POST /v1/auth/login`, then send the cookie with every request (`credentials: "include"` in `fetch`). Writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from this API's origin or `AUTH_WEB_ORIGINS`. Sessions end after `AUTH_SESSION_IDLE_MINUTES` idle (30) or `AUTH_SESSION_MAX_HOURS` (12). * **HTTP Basic:** username and password on every request, e.g. `curl -u analyst: .../v1/stats/publications`. -> [!WARNING] -> Basic auth sends the password with every request. Use it over HTTPS only. - Scope changes apply on the credential's next request. Each endpoint's description names the scopes it needs. -## Managing Credentials +## Managing credentials A credential can only grant scopes it holds, can't change itself, and can't change a credential holding scopes it lacks. These return `403`. `GET /v1/creds/{username}` returns an opaque `ETag`. Send it back unchanged as `If-Match` to change, reset or delete that credential: a missing header gets `428`, a stale one `412`. Responses that change a credential return its new `ETag`. Generated passwords appear only in the response that creates them. -## Managing Platform Adapters +## Managing platform adapters -`/v1/platforms/adapters` lists every installed adapter, with its source, version tag, commit and who last changed it, and takes the same `ETag` and `If-Match` rules as credentials. A disabled adapter disappears from `/v1/platforms` and can't publish or link accounts, but still revokes tokens so users can unlink. Uninstalling is refused with `409` while accounts are linked through the adapter; disable it instead, or remove it with `./publisher.sh platforms remove --force`. +`/v1/platforms/adapters` lists every installed adapter, with its source, version tag, commit and who last changed it, and takes the same `ETag` and `If-Match` rules as credentials. A disabled adapter disappears from `/v1/platforms` and can't publish or link accounts, but still revokes tokens so users can unlink. Uninstalling is refused with `409` while accounts are linked through the adapter; see [Removing](../platforms/manage.md#removing). -Installing (`POST /v1/platforms/adapters`) and updating (`POST /v1/platforms/adapters/{id}/update`) run code from the repository on the server, so they need an administrator and a GitHub repository of an org in `PLATFORMS_GITHUB_ORGS`. Both take an optional `tag`, defaulting to the newest version tag, and return `202` with the job's URL in `Location`. Poll the job until its `state` is `succeeded` or `failed`; its `log` has the end of the git and pip output. An adapter has one job at a time; another gets `409`. +Installing (`POST /v1/platforms/adapters`) and updating (`POST /v1/platforms/adapters/{id}/update`) run code from the repository on the server, so they need an administrator and a repository from an org in `PLATFORMS_GITHUB_ORGS`. Both take an optional `tag`, defaulting to the newest version tag, and return `202` with the job's URL in `Location`. Poll the job until its `state` is `succeeded` or `failed`; its `log` has the end of the git and pip output. An adapter has one job at a time; another gets `409`. An adapter's settings are at `/v1/platforms/adapters/{id}/config`, for administrators only. `GET` lists each setting, with secrets only marked `is_set`, and the inputs its setup takes. `PATCH` changes the settings sent and keeps the rest; `POST .../setup` runs the adapter's setup and replaces them all, with a file input sent as the file's text. `configured` in the adapter list says whether every required setting has a value. -## Managing Gateway Clients +## Managing gateway clients -`/v1/gateway-clients/registry` lists every gateway client, with who last changed it, and takes the same `ETag` and `If-Match` rules as credentials (scopes `gc:read` and `gc:write`). `GET /v1/gateway-clients/registry/suggest?msisdn=` returns the details and PLMN candidates a create would use, to prefill a form for the administrator to confirm; see [how suggest works](gateway-clients.md#how-suggest-works). A disabled client disappears from the public `/v1/gateway-clients`. +`/v1/gateway-clients/registry` lists every gateway client, with who last changed it, and takes the same `ETag` and `If-Match` rules as credentials (scopes `gc:read` and `gc:write`). `GET /v1/gateway-clients/registry/suggest?msisdn=` returns the details and PLMN candidates a create would use, to prefill a form for the administrator to confirm; see [how suggest works](../gateway-clients/manage.md#how-suggest-works). A disabled client disappears from the public `/v1/gateway-clients`. ## Pagination @@ -50,7 +47,7 @@ List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. `since` and `until` take ISO-8601 times, UTC if there's no offset. `since` is inclusive and `until` exclusive. -## Audit Events +## Audit events `GET /v1/audit-events` needs `audit:read`, and each area also needs its read scope: `creds:read` for `auth.*` and `creds.*` events, `platforms:read` for `platforms.*`, `gc:read` for `gateway_clients.*`. Events from areas you can't read are left out. @@ -69,7 +66,7 @@ List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. ## Publishing -`POST /v1/publications` takes JSON and tags payloads with protocol `https`. See [Offline Publishing](../README.md#offline-publishing) for how the protocol and `tag` decide whether offline payloads publish. +`POST /v1/publications` takes JSON and tags payloads with protocol `https`. See [offline publishing](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#offline-publishing) for how the protocol and `tag` decide whether offline payloads publish. It only checks the payload's structure before queueing it. Decryption, payload type and adapter errors happen later, so they're logged on the server, not returned. diff --git a/observability/README.md b/observability/README.md deleted file mode 100644 index ae5b2ab..0000000 --- a/observability/README.md +++ /dev/null @@ -1,185 +0,0 @@ -# Observability - -Self-hosted tracing, metrics, log export (SigNoz) and uptime monitoring -(Uptime Kuma) for Publisher's 5 services. Optional: -`OTEL_EXPORTER_OTLP_ENDPOINT` is blank by default. - -## Setup - -```bash -sudo ./install.sh --setup-observability -# or, against an already-installed Publisher: -sudo ./scripts/setup-observability.sh -``` - -Flags (full list: `scripts/setup-observability.sh --help`): - -| Flag | Does | -| --- | --- | -| `--observability-site-name` (`--site-name`) | Fronts SigNoz with nginx + TLS | -| `--observability-kuma-site-name` (`--kuma-site-name`) | Fronts Kuma with nginx + TLS, different domain from the above | -| `--signoz-email` / `--signoz-password` | SigNoz admin account | -| `--signoz-smtp-host/-port/-username/-password/-from` | SMTP relay for SigNoz's email alert channel | - -Prerequisites: Docker Engine + Compose plugin. - -## SigNoz - -**Automated**: generates `observability/signoz/casting.yaml` from -`casting.yaml.template` (Postgres credentials, ports, SMTP), casts it, -binds the UI/OTLP ports to `127.0.0.1`, fixes the opamp -misconfiguration that keeps the OTLP receiver closed, creates the admin -account. - -**Manual**: - -```bash -curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_INSTALL_DIR=/usr/local/bin bash - -sed \ - -e "s/__POSTGRES_DB__/signoz/" -e "s/__POSTGRES_USER__/signoz/" \ - -e "s/__POSTGRES_PASSWORD__/$(openssl rand -hex 24)/" \ - -e "s/__SIGNOZ_PORT__/8090/" -e "s/__OTLP_GRPC_PORT__/4317/" -e "s/__OTLP_HTTP_PORT__/4318/" \ - -e "s/__SMTP_SMARTHOST__//" -e "s/__SMTP_USERNAME__//" -e "s/__SMTP_PASSWORD__//" -e "s/__SMTP_FROM__//" \ - -e "s|__SIGNOZ_EXTERNAL_URL__|http://localhost:8090|" \ - observability/signoz/casting.yaml.template > observability/signoz/casting.yaml - -foundryctl cast -f observability/signoz/casting.yaml -docker compose \ - -f pours/deployment/compose.yaml \ - -f observability/signoz/docker-compose.override.yml \ - up -d - -until curl -fs http://127.0.0.1:8090/api/v1/version | grep -q setupCompleted; do sleep 2; done -curl -X POST http://127.0.0.1:8090/api/v1/register \ - -H "Content-Type: application/json" \ - -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@relaysms.local","password":"<12+ chars, upper, lower, number, symbol>"}' -``` - -`metastore.kind: postgres` is the default in the template; set it to -`sqlite` for a single-node setup. Retention isn't a `casting.yaml` field: -set it in the UI, under Settings > Workspace > Retention Controls. - -> [!IMPORTANT] -> `casting.yaml` is generated only once. To change a running install, edit -> `observability/signoz/casting.yaml`, then re-run -> `foundryctl cast -f observability/signoz/casting.yaml`. Only the `signoz` -> container is recreated; Postgres and ClickHouse are untouched. - -## Uptime Kuma - -**Automated**: brings up the container, bound to `127.0.0.1:3001` unless -another port is chosen, which it saves to `observability/uptime-kuma/.env`. - -**Manual**: `docker compose -f observability/uptime-kuma/docker-compose.yml up -d` - -Reach it through its own reverse-proxy domain (below) or -`ssh -L 3001:127.0.0.1:3001 @`, then -. Create the admin account on first visit, then add -monitors: - -- **REST**: Monitor Type "HTTP(s)", URL `https:///health`. -- **gRPC**: Monitor Type "GRPC(S) - Keyword". Fields: - - gRPC URL: `:443` - - Enable TLS: on - - Service Name: `grpc.health.v1.Health` - - Method: `check` - - gRPC Body: `{"service":"publisher.v3.Publisher"}` - - Keyword: `SERVING` - - Protobuf: - - ```proto - syntax = "proto3"; - package grpc.health.v1; - - message HealthCheckRequest { - string service = 1; - } - - message HealthCheckResponse { - enum ServingStatus { - UNKNOWN = 0; - SERVING = 1; - NOT_SERVING = 2; - SERVICE_UNKNOWN = 3; - } - ServingStatus status = 1; - } - - service Health { - rpc Check(HealthCheckRequest) returns (HealthCheckResponse); - } - ``` - -- **Worker**: Monitor Type "Push", interval just above 60s. Wire the push - URL Kuma shows into `UPTIME_KUMA_WORKER_PUSH_URL` and - `./manage.sh restart`. -- **SMTP listener**: same as Worker, interval just above 20s, into - `UPTIME_KUMA_SMTP_PUSH_URL`. -- **Beat**: Push monitor, optional. - -## Reverse proxy - -**Automated**: `--site-name`/`--kuma-site-name` on `setup-observability.sh` -write the site files, symlink them, run `certbot`. - -**Manual**: - -```bash -sed -e "s/__SERVER_NAME__//g" \ - observability/nginx-observability.conf.template \ - > /etc/nginx/sites-available/.conf -ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ - -sed -e "s/__SERVER_NAME__//g" \ - observability/nginx-uptime-kuma.conf.template \ - > /etc/nginx/sites-available/.conf -ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ - -nginx -t && systemctl reload nginx -certbot --nginx -d -certbot --nginx -d -``` - -## Turning tracing on/off per service - -> [!NOTE] -> systemd installs don't include `requirements-observability.txt` until you -> install it: - -```bash -cd /opt/relaysms/relaysms-publisher -sudo venv/bin/pip install --quiet -r requirements-observability.txt -``` - -```bash -sudo sed -i 's|^OTEL_EXPORTER_OTLP_ENDPOINT=.*|OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317|' /opt/relaysms/relaysms-publisher/.env -./manage.sh restart -``` - -Blank the value and restart to turn it back off. - -## Alerts - -SigNoz UI, Alerts > New Alert > Logs signal, paste the filter below into the -search bar, set the threshold to count > 0 over a rolling window, attach a -notification channel. `CONTAINS` is a real query-builder operator; wrap the -`OR` group in parentheses since `AND` binds tighter than `OR`. - -- **Publish failures** (`tasks/publication_task.py`): - - ``` - service.name = 'relaysms-publisher-worker' AND (body CONTAINS 'Failed to process payload' OR body CONTAINS 'Failed to publish message' OR body CONTAINS 'An unexpected error occurred during task processing') - ``` - -- **Platform adapter failures** (`publisher/platforms/ipc.py`): - - ``` - service.name = 'relaysms-publisher-worker' AND (body CONTAINS 'Subprocess failed' OR body CONTAINS 'Malformed JSON response' OR body CONTAINS 'Subprocess execution timed out' OR body CONTAINS 'Unexpected failure during IPC invocation') - ``` - -- **Key management failures** (`publisher/keys.py`): - - ``` - body CONTAINS 'Failed to generate server identity keys' OR body CONTAINS 'Token hash missing' - ``` diff --git a/observability/nginx-observability.conf.template b/observability/nginx-observability.conf.template index 9ea4a30..d044aeb 100644 --- a/observability/nginx-observability.conf.template +++ b/observability/nginx-observability.conf.template @@ -1,5 +1,5 @@ # nginx reverse proxy site template for SigNoz. -# Setup: observability/README.md +# Setup: docs/operating/observability.md upstream signoz_ui { server 127.0.0.1:8080; diff --git a/observability/nginx-uptime-kuma.conf.template b/observability/nginx-uptime-kuma.conf.template index 6d1aaba..50e182b 100644 --- a/observability/nginx-uptime-kuma.conf.template +++ b/observability/nginx-uptime-kuma.conf.template @@ -1,5 +1,5 @@ # nginx reverse proxy site template for Uptime Kuma. -# Setup: observability/README.md +# Setup: docs/operating/observability.md upstream uptime_kuma_ui { server 127.0.0.1:3001; diff --git a/observability/signoz/casting.yaml.template b/observability/signoz/casting.yaml.template index 56b305f..4bbd2f0 100644 --- a/observability/signoz/casting.yaml.template +++ b/observability/signoz/casting.yaml.template @@ -3,7 +3,7 @@ # Not run directly: the placeholders below are filled in by # scripts/setup-observability.sh, which writes # observability/signoz/casting.yaml (gitignored, contains real -# credentials). See observability/README.md for the manual equivalent. +# credentials). See docs/operating/observability.md for the manual equivalent. # # Left unset, Foundry defaults the Postgres fields to the literal string # "signoz", so don't cast this template file directly. diff --git a/publisher/gateway_clients/mcc_mnc.py b/publisher/gateway_clients/mcc_mnc.py index a319c41..1c05aa2 100644 --- a/publisher/gateway_clients/mcc_mnc.py +++ b/publisher/gateway_clients/mcc_mnc.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-only """MCC/MNC (PLMN) lookup in the vendored mcc_mnc_table.json snapshot. -See docs/gateway-clients.md. +See docs/gateway-clients/manage.md. """ import json diff --git a/publisher/tasks/celery_app.py b/publisher/tasks/celery_app.py index f7260f4..063b167 100644 --- a/publisher/tasks/celery_app.py +++ b/publisher/tasks/celery_app.py @@ -80,7 +80,7 @@ def make_celery() -> Celery: }, } if celery.worker_heartbeat_url: - # Uptime Kuma push-monitor heartbeat, see observability/README.md + # Uptime Kuma push-monitor heartbeat, see docs/operating/observability.md beat_schedule["worker-heartbeat"] = { "task": "tasks.heartbeat_task.ping_worker_heartbeat", "schedule": 60.0, diff --git a/requirements-observability.txt b/requirements-observability.txt index 8d1c08b..27f8cca 100644 --- a/requirements-observability.txt +++ b/requirements-observability.txt @@ -1,6 +1,6 @@ # OpenTelemetry SDK, exporter and auto-instrumentation, needed only to export # traces, metrics and logs. Without them the app's opentelemetry-api calls do -# nothing. See observability/README.md. +# nothing. See docs/operating/observability.md. # # Includes requirements.txt, since both pull in OTel packages that must # resolve to the same versions. diff --git a/scripts/build-book.sh b/scripts/build-book.sh new file mode 100755 index 0000000..17cf288 --- /dev/null +++ b/scripts/build-book.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-3.0-only +# Builds the manual in docs/ into book/ with pinned mdbook and mdbook-mermaid +# releases, downloaded once into .mdbook/. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/lib.sh +source "$SCRIPT_DIR/lib.sh" + +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +TOOLS_DIR="$ROOT_DIR/.mdbook" +SERVE=false + +MDBOOK_VERSION=0.5.3 +MERMAID_VERSION=0.17.1 +declare -A SHA256=( + ["mdbook-x86_64"]=dfb86d8e20a3fed91bf549aa450b9d7f46a275b700292e050cdfd3171732e7fd + ["mdbook-aarch64"]=428c91cf73fec7494cbefe6e170e12508c9d45d9d096104b7c62d47ff43911c2 + ["mdbook-mermaid-x86_64"]=42aca57eea8521971cc595fe4bb853c5b8b7d9f7138a2ec9f14a0a72c210922d + ["mdbook-mermaid-aarch64"]=b1782ecaf775bfbe069dead6cdd8ac7cc739d5e62d138bd0e800efec6467cf3a +) + +usage() { + cat <<'EOF' +Usage: build-book.sh [OPTIONS] + + --serve Serve the manual on http://localhost:3000 and rebuild on changes + -h, --help Show this help and exit +EOF +} + +parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --serve) + SERVE=true + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + usage >&2 + error "Unknown option: $1" + ;; + esac + done +} + +# $1=tool $2=version $3=GitHub repo +fetch_tool() { + local tool="$1" version="$2" repo="$3" dir="$TOOLS_DIR/$1-$2" key tarball + [ -x "$dir/$tool" ] && return + key="$tool-$(uname -m)" + [ -n "${SHA256[$key]:-}" ] || error "No $tool release for $(uname -m)" + tarball="$(mktemp)" + curl -fsSL -o "$tarball" \ + "https://github.com/$repo/releases/download/v$version/$tool-v$version-$(uname -m)-unknown-linux-musl.tar.gz" + echo "${SHA256[$key]} $tarball" | sha256sum --check --quiet || { + rm -f "$tarball" + error "$tool v$version download doesn't match its sha256" + } + mkdir -p "$dir" + tar -xzf "$tarball" -C "$dir" "$tool" + rm -f "$tarball" + log "Installed $tool v$version" +} + +main() { + parse_args "$@" + fetch_tool mdbook "$MDBOOK_VERSION" rust-lang/mdBook + fetch_tool mdbook-mermaid "$MERMAID_VERSION" badboy/mdbook-mermaid + export PATH="$TOOLS_DIR/mdbook-$MDBOOK_VERSION:$TOOLS_DIR/mdbook-mermaid-$MERMAID_VERSION:$PATH" + + cd "$ROOT_DIR" + # Writes mermaid.min.js and mermaid-init.js from the pinned release. + RUST_LOG=warn mdbook-mermaid install . >/dev/null + if [ "$SERVE" = true ]; then + mdbook serve + else + mdbook build + fi +} + +main "$@" diff --git a/scripts/setup-observability.sh b/scripts/setup-observability.sh index 719e368..42a7d70 100755 --- a/scripts/setup-observability.sh +++ b/scripts/setup-observability.sh @@ -299,7 +299,7 @@ main() { fi case "$create_admin" in n | N | no | NO) - log "Skipping SigNoz admin account. OTLP ingestion stays blocked until one is created, see observability/README.md" + log "Skipping SigNoz admin account. OTLP ingestion stays blocked until one is created, see docs/operating/observability.md" ;; *) [ "$SIGNOZ_EMAIL_SET" = "1" ] || prompt SIGNOZ_EMAIL "SigNoz admin email [$SIGNOZ_EMAIL]: " "$SIGNOZ_EMAIL" @@ -351,7 +351,7 @@ main() { log "SigNoz: http://localhost:$SIGNOZ_PORT Uptime Kuma: http://localhost:$KUMA_PORT" [ -n "$SITE_NAME" ] && [ "$SKIP_NGINX" != "1" ] && log "SigNoz reverse proxy: $signoz_external_url" [ -n "$KUMA_SITE_NAME" ] && [ "$SKIP_NGINX" != "1" ] && log "Uptime Kuma reverse proxy: https://$KUMA_SITE_NAME" - log "Create Uptime Kuma monitors and set retention in SigNoz, see observability/README.md" + log "Create Uptime Kuma monitors and set retention in SigNoz, see docs/operating/observability.md" } main "$@" diff --git a/template.env b/template.env index b177f4b..7711d8c 100644 --- a/template.env +++ b/template.env @@ -146,7 +146,7 @@ API_DOCS_ENABLED=false LOG_LEVEL=info # OpenTelemetry. Optional, blank disables instrumentation. See -# observability/README.md. +# docs/operating/observability.md. OTEL_EXPORTER_OTLP_ENDPOINT= OTEL_EXPORTER_OTLP_PROTOCOL=grpc OTEL_TRACES_SAMPLER=parentbased_traceidratio @@ -155,6 +155,6 @@ OTEL_LOGS_EXPORTER=otlp OTEL_PYTHON_LOG_CORRELATION=true # Uptime Kuma push-monitor heartbeats. Blank disables. See -# observability/README.md. +# docs/operating/observability.md. UPTIME_KUMA_WORKER_PUSH_URL= UPTIME_KUMA_SMTP_PUSH_URL= diff --git a/tests/scripts/test_build_book.py b/tests/scripts/test_build_book.py new file mode 100644 index 0000000..afdddaa --- /dev/null +++ b/tests/scripts/test_build_book.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: GPL-3.0-only + +import os +import stat +import subprocess + +import pytest + + +@pytest.fixture +def build(install, tmp_path): + """Run build-book.sh with stub curl and uname, so nothing is downloaded.""" + stubs = tmp_path / "bin" + stubs.mkdir() + for name, body in { + # Called as: curl -fsSL -o FILE URL + "curl": 'touch "$CURL_CALLED"; echo junk >"$3"', + "uname": 'echo "$ARCH"', + }.items(): + stub = stubs / name + stub.write_text(f"#!/usr/bin/env bash\n{body}\n") + stub.chmod(stub.stat().st_mode | stat.S_IEXEC) + + def run(arch: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", "scripts/build-book.sh"], + cwd=install, + env={ + **os.environ, + "NO_COLOR": "1", + "PATH": f"{stubs}:{os.environ['PATH']}", + "ARCH": arch, + "CURL_CALLED": str(tmp_path / "curl-called"), + }, + capture_output=True, + text=True, + check=False, + ) + + return run + + +class TestBuildBook: + def test_refuses_a_download_that_fails_its_checksum(self, install, build): + result = build("x86_64") + assert result.returncode != 0 + assert "doesn't match its sha256" in result.stderr + assert not (install / ".mdbook" / "mdbook-0.5.3" / "mdbook").exists() + + def test_refuses_an_architecture_without_a_release(self, tmp_path, build): + result = build("riscv64") + assert result.returncode != 0 + assert "No mdbook release for riscv64" in result.stderr + assert not (tmp_path / "curl-called").exists() From 98a97d28f1d7f47326de74f5da006f93f376fcda Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Thu, 22 Oct 2026 19:15:47 +0100 Subject: [PATCH 05/13] feat(install): serve the manual on its own domain with --docs-site-name --- README.md | 1 + ...elaysms-publisher-docs-nginx.conf.template | 30 ++++++++++ install.sh | 57 +++++++++++++++++-- manage.sh | 4 ++ 4 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 deploy/nginx/relaysms-publisher-docs-nginx.conf.template diff --git a/README.md b/README.md index ebd1ac7..4b266c0 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ The installer sets up everything else, including the system packages and Rust. | `--site-name DOMAIN` | Puts nginx with a Let's Encrypt certificate in front; `--skip-nginx` skips it | | `--setup-db mysql\|postgres` | Installs the database server and writes its details to `.env` ([details](INSTALL.md#database)) | | `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker, for heavier load ([details](INSTALL.md#celery-worker-and-beat)) | + | `--docs-site-name DOMAIN` | Serves the manual in `docs/` on its own domain, rebuilt by every `manage.sh update` | | `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](docs/operating/observability.md)) | | `--install-dir PATH`, `--instance-name NAME` | Installs elsewhere, or a second copy on the same host ([details](INSTALL.md#running-multiple-instances)) | diff --git a/deploy/nginx/relaysms-publisher-docs-nginx.conf.template b/deploy/nginx/relaysms-publisher-docs-nginx.conf.template new file mode 100644 index 0000000..bb93a12 --- /dev/null +++ b/deploy/nginx/relaysms-publisher-docs-nginx.conf.template @@ -0,0 +1,30 @@ +# RelaySMS Publisher manual - nginx static site template +# +# Installed to /etc/nginx/sites-available/.conf by install.sh +# --docs-site-name. Only the port 80 block is defined; certbot adds HTTPS. + +server { + listen 80; + listen [::]:80; + server_name __SERVER_NAME__; + + server_tokens off; + root __BOOK_DIR__; + + access_log /var/log/nginx/__SERVER_NAME__.access.log combined buffer=512k flush=1m; + error_log /var/log/nginx/__SERVER_NAME__.error.log warn; + + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "DENY" always; + add_header Referrer-Policy "no-referrer" always; + add_header Cross-Origin-Opener-Policy "same-origin" always; + # mdBook pages and Mermaid diagrams use inline scripts and styles. + add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; frame-ancestors 'none'; base-uri 'self'; form-action 'none'" always; + + gzip on; + gzip_vary on; + gzip_types text/css application/javascript application/json image/svg+xml; + + error_page 404 /404.html; +} diff --git a/install.sh b/install.sh index ee664b5..c6300b9 100755 --- a/install.sh +++ b/install.sh @@ -52,6 +52,7 @@ BRANCH="${BRANCH:-main}" CARGO_BIN="$HOME/.cargo/bin" DEPS_MARKER="/var/lib/relaysms-publisher-deps-installed" SITE_NAME="${SITE_NAME:-}" +DOCS_SITE_NAME="${DOCS_SITE_NAME:-}" LETSENCRYPT_EMAIL="${LETSENCRYPT_EMAIL:-}" SKIP_NGINX="${SKIP_NGINX:-0}" FORCE_DEPS="${FORCE_DEPS:-0}" @@ -101,7 +102,8 @@ Usage: install.sh [OPTIONS] --branch BRANCH Git branch to install (default: main) --site-name DOMAIN Domain to front with nginx + Let's Encrypt --letsencrypt-email EMAIL Email for Let's Encrypt renewal notices - --skip-nginx Skip the nginx/TLS setup entirely + --docs-site-name DOMAIN Domain to serve the manual on with nginx + Let's Encrypt + --skip-nginx Skip the nginx/TLS setup entirely, the manual's site included --force-deps Reinstall system dependencies even if already marked done --setup-db {mysql|postgres} Install and provision a database server, or connect to an existing one --db-existing Use an already-running database server instead of installing one locally @@ -168,6 +170,14 @@ parse_args() { SITE_NAME="${1#*=}" shift ;; + --docs-site-name) + DOCS_SITE_NAME="$2" + shift 2 + ;; + --docs-site-name=*) + DOCS_SITE_NAME="${1#*=}" + shift + ;; --letsencrypt-email) LETSENCRYPT_EMAIL="$2" shift 2 @@ -577,14 +587,49 @@ configure_nginx() { } fi validate_hostname "--site-name" "$site" + SITE_NAME="$site" install_nginx_certbot + install_site "$site" render_nginx_site "$interactive" +} + +render_docs_site() { + sed -e "s/__SERVER_NAME__/$1/g" -e "s|__BOOK_DIR__|$INSTALL_DIR/book|g" \ + "$INSTALL_DIR/deploy/nginx/relaysms-publisher-docs-nginx.conf.template" >"$2" +} + +configure_docs_site() { + if [ "${SKIP_NGINX:-0}" = "1" ]; then + return + fi + + local site="$DOCS_SITE_NAME" interactive=0 + if [ -z "$site" ]; then + # Without --docs-site-name, ask only when the API has a site, since the manual's site needs nginx and certbot. + [ -n "$SITE_NAME" ] || return 0 + interactive=1 + prompt site "Domain name to serve the manual on (e.g. docs.publisher.example.com, blank to skip): " "" + [ -n "$site" ] || { + log "Skipping the manual's site" + return + } + fi + validate_hostname "--docs-site-name" "$site" + + log "Building the manual" + "$INSTALL_DIR/scripts/build-book.sh" + install_nginx_certbot + install_site "$site" render_docs_site "$interactive" +} +# $1=domain $2=function that writes the site file, called as "$2 DOMAIN FILE" $3=1 to ask before certbot and for its email +install_site() { + local site="$1" render="$2" interactive="$3" local conf_dest="/etc/nginx/sites-available/${site}.conf" if [ -f "$conf_dest" ]; then log "nginx site $conf_dest already exists, leaving it untouched" else - render_nginx_site "$site" "$conf_dest" + "$render" "$site" "$conf_dest" log "Wrote $conf_dest" fi ln -sf "$conf_dest" "/etc/nginx/sites-enabled/${site}.conf" @@ -607,12 +652,11 @@ configure_nginx() { ;; esac - local email="${LETSENCRYPT_EMAIL:-}" - if [ -z "$email" ] && [ "$interactive" = "1" ]; then - prompt email "Email for Let's Encrypt renewal notices (optional): " "" + if [ -z "$LETSENCRYPT_EMAIL" ] && [ "$interactive" = "1" ]; then + prompt LETSENCRYPT_EMAIL "Email for Let's Encrypt renewal notices (optional): " "" fi - request_certificate "$site" "$email" || error "certbot failed to obtain a certificate for $site" + request_certificate "$site" "$LETSENCRYPT_EMAIL" || error "certbot failed to obtain a certificate for $site" log "Certificate installed for $site" enable_nginx_http2 "$conf_dest" @@ -817,6 +861,7 @@ main() { run_migrations install_services configure_nginx + configure_docs_site configure_observability log "Installation complete" diff --git a/manage.sh b/manage.sh index e69b8bf..2ddbcbc 100755 --- a/manage.sh +++ b/manage.sh @@ -252,6 +252,10 @@ cmd_update() { systemctl restart "$svc" done systemctl start "$TARGET_UNIT" + # book/ exists only when install.sh set up a site for the manual, so other installs skip the rebuild. + if [ -d book ]; then + ./scripts/build-book.sh + fi [ "$config_ok" = "1" ] || error "Update applied, but .env has errors (above); services using those settings won't start" log "Update complete" } From b4ab70e0602fcf830e7712f22af0d53ca36de7e5 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Thu, 22 Oct 2026 19:33:16 +0100 Subject: [PATCH 06/13] fix(make): make the .env target work with BSD sed --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 726df6a..108e1c7 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,9 @@ venv/bin/python: .env: install -m 600 template.env $@ for key in DATABASE_ENCRYPTION_KEY DATABASE_FIELD_ENCRYPTION_KEY DATA_ENCRYPTION_KEY; do \ - sed -i "s/^$$key=$$/$$key=$$(openssl rand -hex 32)/" $@; \ + sed -i.bak "s/^$$key=$$/$$key=$$(openssl rand -hex 32)/" $@; \ done + rm -f $@.bak ## Generate the gRPC code and the payload-specs bindings. build: protos specs From 2eb2c9b54b3b2e75aea17454706351195485251d Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Thu, 22 Oct 2026 19:37:26 +0100 Subject: [PATCH 07/13] docs: add the quick start and installation pages, replacing INSTALL.md --- .github/workflows/docs.yml | 2 +- INSTALL.md | 270 ------------------------ README.md | 15 +- docs/README.md | 8 +- docs/SUMMARY.md | 22 +- docs/configuration/encryption-keys.md | 12 ++ docs/gateway-clients/manage.md | 20 +- docs/getting-started/quick-start.md | 57 +++++ docs/installation/celery-broker.md | 36 ++++ docs/installation/database.md | 53 +++++ docs/installation/docker.md | 23 ++ docs/installation/manual.md | 84 ++++++++ docs/installation/multiple-instances.md | 21 ++ docs/installation/requirements.md | 12 ++ docs/installation/reverse-proxy.md | 50 +++++ docs/installation/server.md | 84 ++++++++ docs/operating/services.md | 60 ++++++ docs/platforms/manage.md | 35 ++- docs/reference/rest-api.md | 2 +- 19 files changed, 541 insertions(+), 325 deletions(-) delete mode 100644 INSTALL.md create mode 100644 docs/configuration/encryption-keys.md create mode 100644 docs/getting-started/quick-start.md create mode 100644 docs/installation/celery-broker.md create mode 100644 docs/installation/database.md create mode 100644 docs/installation/docker.md create mode 100644 docs/installation/manual.md create mode 100644 docs/installation/multiple-instances.md create mode 100644 docs/installation/requirements.md create mode 100644 docs/installation/reverse-proxy.md create mode 100644 docs/installation/server.md create mode 100644 docs/operating/services.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a8c80b2..5211792 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,7 +28,7 @@ jobs: - name: Check Markdown links uses: lycheeverse/lychee-action@v2 with: - args: --offline --include-fragments --no-progress README.md INSTALL.md CONTRIBUTING.md sdk/README.md "docs/**/*.md" + args: --offline --include-fragments --no-progress README.md CONTRIBUTING.md sdk/README.md "docs/**/*.md" - name: Build run: make book diff --git a/INSTALL.md b/INSTALL.md deleted file mode 100644 index e2d4424..0000000 --- a/INSTALL.md +++ /dev/null @@ -1,270 +0,0 @@ -# Installation Guide - -The [README](README.md#install-on-a-server) covers the one-line installer. This guide covers running several instances, installing by hand, nginx, and every setting. - -## Contents - -- [Running multiple instances](#running-multiple-instances) -- [Manual installation](#manual-installation) -- [Nginx reverse proxy](#nginx-reverse-proxy) -- [Service management](#service-management) -- [The publisher.sh CLI](#the-publishersh-cli) -- [Configuration](#configuration) - -## Running Multiple Instances - -Give each copy its own install directory and instance name: - -```bash -sudo ./install.sh --install-dir /srv/relaysms-acme --instance-name acme -``` - -Its units are namespaced (`relaysms-publisher-acme.target`, `relaysms-publisher-acme-rest.service`, ...), and the `manage.sh` in each install directory manages only its own instance. Re-running `install.sh` remembers the name; a different `--instance-name` is rejected. - -> [!IMPORTANT] -> Set distinct `PORT` and `GRPC_PORT` values in each instance's `.env`. The installer doesn't assign them. - -## Manual Installation - -1. Install the system packages: - - ```bash - sudo apt-get update - sudo apt-get install -y python3 python3-pip python3-venv python3-dev \ - build-essential pkg-config libsqlcipher-dev libmagic1 git make curl - ``` - -2. Install Rust, which builds the payload-specs library: - - ```bash - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source "$HOME/.cargo/env" - ``` - -3. Choose the service user. The installer uses the user running `sudo`; run as root, it creates `relaysms`. To create it yourself: - - ```bash - sudo useradd --system --no-create-home --shell /usr/sbin/nologin relaysms - ``` - -4. Clone the repository with its submodule. The `insteadOf` line fetches the submodule over HTTPS, so no SSH key is needed: - - ```bash - git config --global url."https://github.com/".insteadOf "git@github.com:" - sudo git clone --recurse-submodules \ - https://github.com/smswithoutborders/RelaySMS-Publisher.git \ - /opt/relaysms/relaysms-publisher - cd /opt/relaysms/relaysms-publisher - ``` - -5. Create the virtualenv and build the gRPC code and payload-specs library: - - ```bash - python3 -m venv venv - venv/bin/pip install --upgrade pip - venv/bin/pip install -r requirements.txt - make build - ``` - -6. Create `.env`, then fill it in (see [Configuration](#configuration)): - - ```bash - cp template.env .env - sudo chown root:relaysms .env - sudo chmod 640 .env - ``` - -7. Create the data directories. By default the SQLite database, Celery files and adapters all live under `data/`: - - ```bash - SERVICE_USER=relaysms # or your own user - sudo mkdir -p data/platforms/adapters data/platforms/venvs data/platforms/config data/platforms/state - sudo chown -R "$SERVICE_USER": data && sudo chmod -R 750 data - ``` - - If `.env` moves any of these paths elsewhere, create their parent directories instead: `SQLITE_DATABASE_PATH`, `CELERY_BROKER_DB_PATH`, `CELERY_RESULT_DB_PATH`, `CELERY_BEAT_SCHEDULE_PATH`, `PLATFORMS_ADAPTERS_DIR`, `PLATFORMS_ADAPTERS_VENV_DIR`, `PLATFORMS_ADAPTERS_CONFIG_DIR`, `PLATFORMS_ADAPTERS_STATE_DIR`. - -8. Apply the database migrations: - - ```bash - make migrate - ``` - -9. Install and start the systemd units. They have a `__RW_PATHS__` placeholder for the directories the services may write to: - - ```bash - RW_PATHS="$(pwd)/data" - for unit in deploy/systemd/*; do - sed -e "s/User=relaysms/User=$SERVICE_USER/" \ - -e "s|/opt/relaysms/relaysms-publisher|$(pwd)|g" \ - -e "s|__RW_PATHS__|$RW_PATHS|" \ - "$unit" | sudo tee "/etc/systemd/system/$(basename "$unit")" >/dev/null - done - sudo systemctl daemon-reload - sudo systemctl enable --now relaysms-publisher.target - ``` - -> [!WARNING] -> The services run with `ProtectSystem=strict` and `ProtectHome=true`, so everything outside `ReadWritePaths` is read-only. If `.env` moves a path out of `data/`, add its parent directory to `RW_PATHS`, or writes to it fail. - -## Nginx Reverse Proxy - -`install.sh` can put nginx with a Let's Encrypt certificate in front, using [the template](deploy/nginx/relaysms-publisher-nginx.conf.template). It proxies `/` to `PORT` (REST) and `/publisher.v3.Publisher` to `GRPC_PORT` (gRPC). - -| Flag | Does | -| --- | --- | -| `--site-name DOMAIN` | Sets up nginx for this domain without asking | -| `--letsencrypt-email EMAIL` | Email for Certbot renewal notices (optional) | -| `--skip-nginx` | Skips nginx, even when run interactively | - -```bash -curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ - sudo bash -s -- --site-name publisher.example.com --letsencrypt-email you@example.com -``` - -Re-running it leaves an existing site file alone and skips Certbot when the domain already has a certificate. To set it up by hand: - -```bash -sudo apt-get install -y nginx certbot python3-certbot-nginx -sudo sed -e "s/__SERVER_NAME__/publisher.example.com/g" \ - -e "s/__REST_PORT__/16000/g" -e "s/__GRPC_PORT__/6000/g" \ - deploy/nginx/relaysms-publisher-nginx.conf.template | - sudo tee /etc/nginx/sites-available/publisher.example.com.conf >/dev/null -sudo ln -s /etc/nginx/sites-available/publisher.example.com.conf /etc/nginx/sites-enabled/ -sudo nginx -t && sudo systemctl reload nginx -sudo certbot --nginx -d publisher.example.com --redirect -``` - -## Service Management - -The [README](README.md#manage-the-services) lists the `manage.sh` commands. Run them from the install directory with `sudo`. - -`logs` filters with `-u/--unit` (rest, grpc, worker, beat or smtp; repeatable), `-n/--lines` and `-s/--since`, and `--no-follow` prints and exits: - -```bash -./manage.sh logs --unit rest --unit grpc --since "1 hour ago" --lines 200 -``` - -> [!IMPORTANT] -> Add `--migrate` to `update` when a release has migrations. They run after the dependencies are reinstalled and before the services restart. - -`update` also checks `.env`. Services restart either way, since each fails only on the settings it uses, but the command exits with an error when a setting is invalid. - -`nginx` keeps the previous site file as `.conf.bak` and restores it if the new one fails. Run it after an update that changes the template. - -## The publisher.sh CLI - -`./publisher.sh` runs the admin CLI (`creds`, `platforms`, `gateway-clients`, `config`) from the install directory, with the venv and `.env`, as the service user. - -> [!IMPORTANT] -> Use it instead of `python3 -m publisher`. Run as another user, the CLI can leave adapter files the services can't read. - -```bash -./publisher.sh --help # every command -./publisher.sh env # resolved paths and service user -./publisher.sh shell # a shell as the service user with .env loaded -``` - -[Platform adapters](docs/platforms/manage.md), [gateway clients](docs/gateway-clients/manage.md) and [credentials](README.md#credentials) each have their own guide. Observability is optional; see [docs/operating/observability.md](docs/operating/observability.md). - -## Configuration - -Settings live in `.env`; `template.env` documents each one. The common ones: - -### Server - -```bash -HOST=127.0.0.1 -PORT=16000 -GRPC_HOST=127.0.0.1 -GRPC_PORT=6000 -GRPC_TLS_ENABLED=false -``` - -### Database - -SQLite is the default: - -```bash -DATABASE_DIALECT=sqlite -SQLITE_DATABASE_PATH=data/relaysms.db -``` - -For MySQL, MariaDB or PostgreSQL, let a script install the server and create a database and user. Both are safe to re-run and write the connection details to `.env`: - -```bash -sudo ./install.sh --setup-db postgres # during install -sudo ./scripts/setup-postgres.sh --db-name relaysms --db-user relaysms # afterwards -sudo ./scripts/setup-mysql.sh --db-name relaysms --db-user relaysms -``` - -`--db-password PASS` sets the password instead of generating one. To use a database server you already have, add `--db-existing` with its details. That only checks the connection and writes `.env`; it never changes the server: - -```bash -sudo ./install.sh --setup-db postgres --db-existing \ - --db-host db.example.com --db-port 5432 \ - --db-name relaysms --db-user relaysms --db-password 'your-password' -``` - -Or set it by hand: - -```bash -DATABASE_DIALECT=mysql # or postgres -MYSQL_HOST=127.0.0.1 # POSTGRES_HOST, POSTGRES_PORT, ... for postgres -MYSQL_PORT=3306 -MYSQL_USER=your_user -MYSQL_PASSWORD=your_password -MYSQL_DATABASE=relaysms_publisher -``` - -### Encryption - -The installer generates these with `openssl rand -hex 32`: - -| Setting | Encrypts | -| --- | --- | -| `DATA_ENCRYPTION_KEY` | Every private key, always | -| `DATABASE_ENCRYPTION_KEY` | The whole database at rest, when `DATABASE_ENCRYPTION_ENABLED=true` (SQLite only) | -| `DATABASE_FIELD_ENCRYPTION_KEY` | Sensitive fields, when `DATABASE_FIELD_ENCRYPTION_ENABLED=true` | - -> [!CAUTION] -> Back the keys up with the database. Data encrypted with a lost or changed key can't be read again. - -### Celery (Worker and Beat) - -The SQLite broker suits light load. For more, use RabbitMQ: - -```bash -sudo ./install.sh --setup-broker rabbitmq # during install -sudo ./scripts/setup-rabbitmq.sh --broker-vhost relaysms --broker-user relaysms # afterwards -``` - -`--broker-password PASS` sets the password. `--broker-existing` (with `--broker-host`) uses a broker you already have: it checks the credentials against the management API (port `15672`, or `--broker-mgmt-port`) and changes nothing. By hand: - -```bash -CELERY_BROKER_TYPE=sqlite # sqlite | redis | rabbitmq -CELERY_BROKER_DB_PATH=data/celery_broker.db -CELERY_RESULT_DB_PATH=data/celery_results.db -# CELERY_REDIS_URL=redis://localhost:6379/0 -# CELERY_RABBITMQ_URL=amqp://user:pass@localhost:5672// -CELERY_BEAT_SCHEDULE_PATH=data/celerybeat-schedule -``` - -### Platform Adapters - -```bash -PLATFORMS_ADAPTERS_DIR=data/platforms/adapters -PLATFORMS_ADAPTERS_VENV_DIR=data/platforms/venvs -PLATFORMS_ADAPTERS_CONFIG_DIR=data/platforms/config # per-adapter credentials.json -PLATFORMS_ADAPTERS_STATE_DIR=data/platforms/state # per-adapter databases and other files -PLATFORMS_GITHUB_ORGS= # orgs administrators may install from over the API -``` - -### Files - -| Path | Holds | -| --- | --- | -| `/opt/relaysms/relaysms-publisher/` | The install | -| `.env` | Configuration (`root:relaysms`, 640) | -| `data/` | SQLite database, Celery files and adapters (service user, 750) | -| `/etc/systemd/system/relaysms-publisher*` | Service units | diff --git a/README.md b/README.md index 4b266c0..0c59b08 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,11 @@ The installer sets up everything else, including the system packages and Rust. | Flag | Does | | --- | --- | | `--site-name DOMAIN` | Puts nginx with a Let's Encrypt certificate in front; `--skip-nginx` skips it | - | `--setup-db mysql\|postgres` | Installs the database server and writes its details to `.env` ([details](INSTALL.md#database)) | - | `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker, for heavier load ([details](INSTALL.md#celery-worker-and-beat)) | + | `--setup-db mysql\|postgres` | Installs the database server and writes its details to `.env` ([details](docs/installation/database.md)) | + | `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker, for heavier load ([details](docs/installation/celery-broker.md)) | | `--docs-site-name DOMAIN` | Serves the manual in `docs/` on its own domain, rebuilt by every `manage.sh update` | | `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](docs/operating/observability.md)) | - | `--install-dir PATH`, `--instance-name NAME` | Installs elsewhere, or a second copy on the same host ([details](INSTALL.md#running-multiple-instances)) | + | `--install-dir PATH`, `--instance-name NAME` | Installs elsewhere, or a second copy on the same host ([details](docs/installation/multiple-instances.md)) | `install.sh --help` lists every flag. @@ -93,7 +93,7 @@ The installer sets up everything else, including the system packages and Rust. curl http://127.0.0.1:16000/health ``` -[INSTALL.md](INSTALL.md) covers installing by hand and every setting. +[Installation](docs/installation/server.md) covers the installer, installing by hand, databases and nginx. ## Manage the services @@ -111,7 +111,7 @@ From the install directory: | `sudo ./manage.sh nginx [DOMAIN]` | Re-render the nginx site and reattach or obtain its certificate | | `sudo ./manage.sh uninstall` | Remove the services and the install directory | -See [Service Management](INSTALL.md#service-management) for log filters and what `update` and `nginx` do. +See [Services](docs/operating/services.md) for log filters and what `update` and `nginx` do. ## Run with Docker @@ -137,7 +137,7 @@ See [Service Management](INSTALL.md#service-management) for log filters and what The container applies pending migrations, then starts every service. It listens on `PORT` (16000) and `GRPC_PORT` (6000) from `.env`, and keeps its data in `./data`. > [!CAUTION] -> Back up `.env` with `./data`: its [encryption keys](INSTALL.md#encryption) are the only way to read the data. +> Back up `.env` with `./data`: its [encryption keys](docs/configuration/encryption-keys.md) are the only way to read the data. ## Development @@ -165,7 +165,7 @@ Settings live in `.env`. `template.env` lists every setting with its default and | Settings | Documented in | | --- | --- | -| Server, database, encryption, Celery, adapters | [INSTALL.md](INSTALL.md#configuration) | +| Database, Celery broker, encryption keys | [Database](docs/installation/database.md), [Celery broker](docs/installation/celery-broker.md), [Encryption keys](docs/configuration/encryption-keys.md) | | REST API login and sessions (`AUTH_*`) | [REST API](docs/reference/rest-api.md#authentication) | | SMTP transport (`SMTP_*`) | [SMTP transport](docs/configuration/smtp-transport.md#configuration) | | Tracing, metrics and logs (`OTEL_*`) | [Observability](docs/operating/observability.md) | @@ -224,7 +224,6 @@ Credentials log in to the REST API, each with its own scopes. One holding every ## Documentation - [Manual](docs/README.md): installing, configuring and running the Publisher -- [Installation guide](INSTALL.md): installing by hand and every setting - [REST API](docs/reference/rest-api.md): publishing, authentication and the [OpenAPI reference](docs/openapi.json) - [gRPC API](docs/reference/grpc-api.md): linking accounts and syncing keys - [SMTP transport](docs/configuration/smtp-transport.md): publishing payloads sent by email diff --git a/docs/README.md b/docs/README.md index 1cceecd..5b027f0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -36,16 +36,12 @@ flowchart TB 3. The gateway client posts the payload to the REST API, or emails it to a mailbox the Publisher reads. 4. A Celery worker decrypts the payload and publishes it through the platform's adapter. -## What you need - -- A server running Ubuntu 24.04 or Debian 13, with a domain name pointing at it -- Developer credentials from each platform you offer, such as a Google Cloud OAuth client for Gmail -- At least one gateway client: a phone number whose SMS reach the REST API or a mailbox - ## Where to start | To | Read | | --- | --- | +| Try the Publisher on your machine | [Quick start](getting-started/quick-start.md) | +| Run it in production | [Requirements](installation/requirements.md), then [Install on a server](installation/server.md) | | Offer a platform | [Managing adapters](platforms/manage.md) | | Relay SMS to the Publisher | [Managing gateway clients](gateway-clients/manage.md) | | Build a client | [gRPC API](reference/grpc-api.md) and [REST API](reference/rest-api.md) | diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index be849a1..4741dc6 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -4,23 +4,23 @@ # Getting started -- [Quick start]() +- [Quick start](getting-started/quick-start.md) # Installation -- [Requirements]() -- [Install on a server]() -- [Install with Docker]() -- [Install by hand]() -- [Database]() -- [Celery broker]() -- [Reverse proxy and TLS]() -- [Multiple instances]() +- [Requirements](installation/requirements.md) +- [Install on a server](installation/server.md) +- [Install with Docker](installation/docker.md) +- [Install by hand](installation/manual.md) +- [Database](installation/database.md) +- [Celery broker](installation/celery-broker.md) +- [Reverse proxy and TLS](installation/reverse-proxy.md) +- [Multiple instances](installation/multiple-instances.md) # Configuration - [Settings reference]() -- [Encryption keys]() +- [Encryption keys](configuration/encryption-keys.md) - [Offline publishing]() - [SMTP transport](configuration/smtp-transport.md) @@ -40,7 +40,7 @@ # Operating -- [Services]() +- [Services](operating/services.md) - [Credentials and access]() - [Upgrading]() - [Backup and restore]() diff --git a/docs/configuration/encryption-keys.md b/docs/configuration/encryption-keys.md new file mode 100644 index 0000000..d6d8da1 --- /dev/null +++ b/docs/configuration/encryption-keys.md @@ -0,0 +1,12 @@ +# Encryption keys + +Three keys in `.env` protect the data. The installer and `make .env` generate them. Each is 64 hex characters; `openssl rand -hex 32` makes one, and `manage.sh check` reports a missing or malformed key. + +| Setting | Protects | Used | +| --- | --- | --- | +| `DATA_ENCRYPTION_KEY` | The server's private keys, and the keyed hash that replaces each sender's address | Always | +| `DATABASE_FIELD_ENCRYPTION_KEY` | Linked accounts' platform tokens | When `DATABASE_FIELD_ENCRYPTION_ENABLED=true` (the default) | +| `DATABASE_ENCRYPTION_KEY` | The whole SQLite database file (SQLCipher) | When `DATABASE_ENCRYPTION_ENABLED=true` (the default) and the database is SQLite | + +> [!CAUTION] +> Back up `.env` with the database, and never change a key once the install holds data. Data encrypted with a lost or changed key can't be read again, and a new `DATA_ENCRYPTION_KEY` also gives every sender a new hash. diff --git a/docs/gateway-clients/manage.md b/docs/gateway-clients/manage.md index a135597..1ba7032 100644 --- a/docs/gateway-clients/manage.md +++ b/docs/gateway-clients/manage.md @@ -5,16 +5,16 @@ Gateway clients are the phone numbers that relay SMS to this server. The enabled ## Commands ```bash -./publisher.sh gateway-clients suggest # details and PLMN candidates; saves nothing -./publisher.sh gateway-clients create --msisdn --protocols [--country ] [--operator ] [--operator-code ] -./publisher.sh gateway-clients list [--msisdn ] [--country ] [--operator ] -./publisher.sh gateway-clients update [--country ] [--operator ] [--operator-code ] [--protocols ] -./publisher.sh gateway-clients disable # hide it from the public list -./publisher.sh gateway-clients enable -./publisher.sh gateway-clients delete -./publisher.sh gateway-clients countries -./publisher.sh gateway-clients operators --country -./publisher.sh gateway-clients mcc-mnc list [--country-code ] [--network ] [--iso ] # search the PLMN table +sudo ./publisher.sh gateway-clients suggest # details and PLMN candidates; saves nothing +sudo ./publisher.sh gateway-clients create --msisdn --protocols [--country ] [--operator ] [--operator-code ] +sudo ./publisher.sh gateway-clients list [--msisdn ] [--country ] [--operator ] +sudo ./publisher.sh gateway-clients update [--country ] [--operator ] [--operator-code ] [--protocols ] +sudo ./publisher.sh gateway-clients disable # hide it from the public list +sudo ./publisher.sh gateway-clients enable +sudo ./publisher.sh gateway-clients delete +sudo ./publisher.sh gateway-clients countries +sudo ./publisher.sh gateway-clients operators --country +sudo ./publisher.sh gateway-clients mcc-mnc list [--country-code ] [--network ] [--iso ] # search the PLMN table ``` `create` fills in any of `--country`, `--operator` and `--operator-code` you leave out with what `suggest` finds. It fills the PLMN code (MCC+MNC) only when every candidate shares it; otherwise run `suggest`, pick a candidate and pass it with `--operator-code`. diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md new file mode 100644 index 0000000..cf012c6 --- /dev/null +++ b/docs/getting-started/quick-start.md @@ -0,0 +1,57 @@ +# Quick start + +Runs the Publisher on your machine with Docker, to try it before installing it on a server. You need Git, Docker with the Compose plugin, `make` and `openssl`. + +1. Clone the repository with its submodule: + + ```bash + git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git + cd RelaySMS-Publisher + ``` + +2. Create `.env` with fresh encryption keys: + + ```bash + make .env + ``` + +3. Build and start. The first build compiles the payload library in Rust and takes several minutes: + + ```bash + docker compose up -d --build + ``` + +4. Check that it's up: + + ```bash + curl http://localhost:16000/health + ``` + + You should see `{"status":"ok"}`. If the connection fails, the container is still applying migrations; `docker compose logs -f` shows its progress. + +5. Create an administrator credential: + + ```bash + docker compose exec publisher python -m publisher creds create --username admin --administrator + ``` + + > [!IMPORTANT] + > The password is printed once. Store it before closing the terminal. + +6. Use it on the REST API: + + ```bash + curl -u admin: http://localhost:16000/v1/auth/me + ``` + + You should see `"username":"admin"` and `"administrator":true`. + +To stop the Publisher, run `docker compose down`. To start again from scratch, also run `sudo rm -rf data .env`. + +## Next steps + +- [Add a platform](../platforms/manage.md) +- [Register a gateway client](../gateway-clients/manage.md) +- [Install on a server](../installation/server.md) for production + +Those pages run the admin CLI as `sudo ./publisher.sh`; with Docker, use `docker compose exec publisher python -m publisher` instead. diff --git a/docs/installation/celery-broker.md b/docs/installation/celery-broker.md new file mode 100644 index 0000000..4b68f61 --- /dev/null +++ b/docs/installation/celery-broker.md @@ -0,0 +1,36 @@ +# Celery broker + +The REST API and the email listener queue publications for the Celery worker through a broker. + +| `CELERY_BROKER_TYPE` | Use it for | Worker concurrency by default | +| --- | --- | --- | +| `sqlite` (default) | Light load. Needs no server. | 1 | +| `rabbitmq` | Heavier load | 4 | +| `redis` | Heavier load, with a Redis server you already run | 4 | + +`CELERY_WORKER_CONCURRENCY` overrides the concurrency. + +## RabbitMQ + +A setup script installs RabbitMQ, creates a vhost and user, and writes the broker settings to `.env`: + +```bash +sudo ./install.sh --setup-broker rabbitmq # during install +sudo ./scripts/setup-rabbitmq.sh --broker-vhost relaysms --broker-user relaysms # afterwards +``` + +`--broker-password PASS` sets the password instead of generating one. To use a broker you already have, add `--existing` (`--broker-existing` on `install.sh`) with `--broker-host`. The script checks the credentials against the management API (port `15672`, or `--broker-mgmt-port`) and changes nothing on the broker. + +## By hand + +```bash +CELERY_BROKER_TYPE=rabbitmq +CELERY_RABBITMQ_URL=amqp://relaysms:your-password@localhost:5672/relaysms +``` + +```bash +CELERY_BROKER_TYPE=redis +CELERY_REDIS_URL=redis://localhost:6379/0 +``` + +After either, restart the services with `sudo ./manage.sh restart`. diff --git a/docs/installation/database.md b/docs/installation/database.md new file mode 100644 index 0000000..e0a213d --- /dev/null +++ b/docs/installation/database.md @@ -0,0 +1,53 @@ +# Database + +| Dialect | Use it for | Encryption at rest | +| --- | --- | --- | +| SQLite (default) | One server with light load | SQLCipher, with `DATABASE_ENCRYPTION_KEY` | +| MySQL 8.0.28+ or MariaDB | Heavier load, or a database on another host | The server's own (TDE) | +| PostgreSQL 12+ | Heavier load, or a database on another host | The disk's | + +Sensitive fields are also encrypted in every dialect; see [Encryption keys](../configuration/encryption-keys.md). + +## MySQL, MariaDB or PostgreSQL + +A setup script installs the server, creates the database and user, and writes the connection details to `.env`. Running it again without `--db-password` sets a new password. The MySQL script installs `mysql-server`, which only Ubuntu ships; on Debian, set up MariaDB yourself and pass `--db-existing`. + +```bash +sudo ./install.sh --setup-db postgres # during install +sudo ./scripts/setup-postgres.sh --db-name relaysms --db-user relaysms # afterwards +sudo ./scripts/setup-mysql.sh --db-name relaysms --db-user relaysms +``` + +`--db-password PASS` sets the password instead of generating one. + +To use a server you already have, pass its details. The script only checks the connection and writes `.env`; it never changes the server: + +```bash +sudo ./install.sh --setup-db postgres --db-existing \ + --db-host db.example.com --db-port 5432 \ + --db-name relaysms --db-user relaysms --db-password 'your-password' + +sudo ./scripts/setup-postgres.sh --existing \ + --db-host db.example.com --db-name relaysms --db-user relaysms --db-password 'your-password' +``` + +Or set it by hand: + +```bash +DATABASE_DIALECT=postgres # or mysql, for MySQL and MariaDB +POSTGRES_HOST=127.0.0.1 # MYSQL_HOST, MYSQL_PORT, ... for mysql +POSTGRES_PORT=5432 +POSTGRES_USER=relaysms +POSTGRES_PASSWORD=your-password +POSTGRES_DATABASE=relaysms +``` + +Then apply the migrations and restart: + +```bash +sudo ./manage.sh migrate +sudo ./manage.sh restart +``` + +> [!WARNING] +> The new database starts empty. Switching doesn't copy data from SQLite. diff --git a/docs/installation/docker.md b/docs/installation/docker.md new file mode 100644 index 0000000..8672eaf --- /dev/null +++ b/docs/installation/docker.md @@ -0,0 +1,23 @@ +# Install with Docker + +`docker-compose.yml` runs every service in one container, with its data in `./data` and its settings in `.env` (`template.env` describes each one): + +```bash +git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git +cd RelaySMS-Publisher +make .env +docker compose up -d --build +``` + +The container applies pending migrations, then starts every service. It publishes `PORT` (16000) and `GRPC_PORT` (6000) on every interface. To serve it publicly, bind them to `127.0.0.1` in `docker-compose.yml` and put [nginx with TLS](reverse-proxy.md#by-hand) in front. + +Run the admin CLI with `docker compose exec publisher python -m publisher `. + +Back up `.env` with `./data`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. + +## Updating + +```bash +git pull --recurse-submodules +docker compose up -d --build +``` diff --git a/docs/installation/manual.md b/docs/installation/manual.md new file mode 100644 index 0000000..ddafca8 --- /dev/null +++ b/docs/installation/manual.md @@ -0,0 +1,84 @@ +# Install by hand + +These are the steps the [installer](server.md) runs, for when you need to change one. Run them in a root shell (`sudo -i`). + +1. Install the system packages: + + ```bash + apt-get update + apt-get install -y python3 python3-pip python3-venv python3-dev \ + build-essential pkg-config libsqlcipher-dev libmagic1 openssl git make curl + ``` + +2. Install Rust, which builds the payload library: + + ```bash + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source "$HOME/.cargo/env" + ``` + +3. Create the service user: + + ```bash + useradd --system --no-create-home --shell /usr/sbin/nologin relaysms + ``` + +4. Clone the repository with its submodule: + + ```bash + git clone --recurse-submodules \ + https://github.com/smswithoutborders/RelaySMS-Publisher.git \ + /opt/relaysms/relaysms-publisher + cd /opt/relaysms/relaysms-publisher + ``` + +5. Create the virtualenv and build the gRPC code and payload library: + + ```bash + python3 -m venv venv + venv/bin/pip install --upgrade pip + venv/bin/pip install -r requirements.txt + make build + ``` + +6. Create `.env` with fresh encryption keys, then set what you need (`template.env` describes each one): + + ```bash + make .env + chown root:relaysms .env + chmod 640 .env + ``` + +7. Create the data directories. By default the SQLite database, Celery files and adapters all live under `data/`: + + ```bash + mkdir -p data/platforms/{adapters,venvs,config,state} + chown -R relaysms: data + chmod -R 750 data + ``` + + If `.env` moves any of these paths out of `data/` (`SQLITE_DATABASE_PATH`, `CELERY_BROKER_DB_PATH`, `CELERY_RESULT_DB_PATH`, `CELERY_BEAT_SCHEDULE_PATH` and the four `PLATFORMS_ADAPTERS_*` directories), create its parent directory and add it to `RW_PATHS` in step 9. Everything else is read-only to the services. + + For MySQL, PostgreSQL or RabbitMQ, run its setup script now ([Database](database.md), [Celery broker](celery-broker.md)). + +8. Check the configuration and apply the database migrations, as the service user: + + ```bash + sudo -u relaysms venv/bin/python -m publisher config check + sudo -u relaysms venv/bin/python -m alembic upgrade head + ``` + +9. Install and start the systemd units. `RW_PATHS` lists the directories the services may write to: + + ```bash + RW_PATHS="$(pwd)/data" + for unit in deploy/systemd/*; do + sed -e "s|/opt/relaysms/relaysms-publisher|$(pwd)|g" \ + -e "s|__RW_PATHS__|$RW_PATHS|" \ + "$unit" > "/etc/systemd/system/$(basename "$unit")" + done + systemctl daemon-reload + systemctl enable --now relaysms-publisher.target + ``` + +Then set up [nginx and TLS](reverse-proxy.md) and continue with [After installing](server.md#after-installing). diff --git a/docs/installation/multiple-instances.md b/docs/installation/multiple-instances.md new file mode 100644 index 0000000..e3fbdcb --- /dev/null +++ b/docs/installation/multiple-instances.md @@ -0,0 +1,21 @@ +# Multiple instances + +To run a second Publisher on the same host, give it its own install directory, instance name and ports: + +1. Install it without nginx. When it shares a RabbitMQ or database server with the first instance, also pass `--broker-vhost acme` or `--db-name acme`: + + ```bash + sudo ./install.sh --install-dir /srv/relaysms-acme --instance-name acme --skip-nginx + ``` + +2. Set free `PORT` and `GRPC_PORT` values in `/srv/relaysms-acme/.env`. The installer gives every instance 16000 and 6000. + +3. Restart it, then put nginx in front: + + ```bash + cd /srv/relaysms-acme + sudo ./manage.sh restart + sudo ./manage.sh nginx acme.example.com + ``` + +Its units are named after the instance (`relaysms-publisher-acme.target`, `relaysms-publisher-acme-rest.service`, ...), and the `manage.sh` in each install directory manages only its own instance. Running `install.sh` again on that directory reuses the name; a different `--instance-name` is refused. diff --git a/docs/installation/requirements.md b/docs/installation/requirements.md new file mode 100644 index 0000000..a036925 --- /dev/null +++ b/docs/installation/requirements.md @@ -0,0 +1,12 @@ +# Requirements + +| Need | Details | +| --- | --- | +| Operating system | Ubuntu 24.04 or Debian 13, or any Linux host with Docker | +| Python | 3.12 or newer. The installer installs it, with the other system packages and Rust. | +| Database | SQLite (the default), MySQL 8.0.28 or newer, MariaDB, or PostgreSQL 12 or newer. See [Database](database.md). | +| Domain | A DNS name pointing at the server, for the TLS certificate. See [Reverse proxy and TLS](reverse-proxy.md). | +| Inbound ports | 80 and 443 for nginx. On a server install, the services listen on `127.0.0.1` only. | +| Outbound access | `github.com`, `pypi.org`, the Rust installer and the system's package mirrors to install the Publisher and its adapters; Let's Encrypt for certificates; each platform's API to publish | +| Platform credentials | From each platform you offer, e.g. a Google Cloud OAuth client for Gmail. See [Managing adapters](../platforms/manage.md). | +| Gateway client | At least one phone number that relays SMS to the REST API or a mailbox. See [Managing gateway clients](../gateway-clients/manage.md). | diff --git a/docs/installation/reverse-proxy.md b/docs/installation/reverse-proxy.md new file mode 100644 index 0000000..2297561 --- /dev/null +++ b/docs/installation/reverse-proxy.md @@ -0,0 +1,50 @@ +# Reverse proxy and TLS + +The services listen on `127.0.0.1`. nginx serves them on port 443 with a Let's Encrypt certificate, from [the site template](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/deploy/nginx/relaysms-publisher-nginx.conf.template): + +| Path | Goes to | +| --- | --- | +| `/publisher.v3.Publisher`, `/grpc.health.v1.Health` | gRPC, on `GRPC_PORT` | +| Everything else | REST, on `PORT` | + +The site also rate-limits logins and requests carrying HTTP Basic credentials. + +## With the installer + +Pass `--site-name DOMAIN` to the [installer](server.md#run-the-installer). Running it again leaves an existing site file alone, and skips Certbot when the domain already has a certificate; use [`manage.sh nginx`](#on-an-existing-install) to render the site again. + +## On an existing install + +`manage.sh nginx` renders the site from the template, obtains or reattaches its certificate, and turns on HTTP/2, which gRPC needs. Run it to add nginx later, or after an update that changes the template. It keeps the previous file as `.conf.bak` and restores it if the new one fails. + +```bash +sudo apt-get install -y nginx certbot python3-certbot-nginx +sudo ./manage.sh nginx publisher.example.com +``` + +## By hand + +With nginx and Certbot installed as above: + +```bash +sudo sed -e "s/__SERVER_NAME__/publisher.example.com/g" \ + -e "s/__REST_PORT__/16000/g" -e "s/__GRPC_PORT__/6000/g" \ + deploy/nginx/relaysms-publisher-nginx.conf.template | + sudo tee /etc/nginx/sites-available/publisher.example.com.conf >/dev/null +sudo ln -s /etc/nginx/sites-available/publisher.example.com.conf /etc/nginx/sites-enabled/ +sudo nginx -t && sudo systemctl reload nginx +sudo certbot --nginx -d publisher.example.com --redirect +``` + +Then turn on HTTP/2 in the HTTPS block Certbot added: add `http2 on;` after `listen 443 ssl;` (nginx 1.25.1 or newer), or change it to `listen 443 ssl http2;`. Reload nginx. + +## gRPC TLS without nginx + +The gRPC server can serve TLS itself. The REST API can't, so it still needs a reverse proxy. + +```bash +GRPC_HOST=0.0.0.0 +GRPC_TLS_ENABLED=true +GRPC_TLS_CERT_FILE=/path/to/fullchain.pem +GRPC_TLS_KEY_FILE=/path/to/privkey.pem +``` diff --git a/docs/installation/server.md b/docs/installation/server.md new file mode 100644 index 0000000..e194a69 --- /dev/null +++ b/docs/installation/server.md @@ -0,0 +1,84 @@ +# Install on a server + +The installer sets up the Publisher on Ubuntu 24.04 or Debian 13 under systemd, in `/opt/relaysms/relaysms-publisher`. Check the [requirements](requirements.md) first. + +## Run the installer + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | sudo bash +``` + +It asks about each option you don't pass as a flag. To choose up front, pass flags after `bash -s --`: + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ + sudo bash -s -- --site-name publisher.example.com --setup-db postgres +``` + +| Flag | Does | +| --- | --- | +| `--site-name DOMAIN` | Puts nginx with a Let's Encrypt certificate in front ([details](reverse-proxy.md)) | +| `--letsencrypt-email EMAIL` | Email for Let's Encrypt renewal notices | +| `--docs-site-name DOMAIN` | Serves this manual on its own domain ([details](#serving-the-manual)) | +| `--skip-nginx` | Skips nginx and certificates entirely | +| `--setup-db mysql\|postgres` | Uses MySQL or PostgreSQL instead of SQLite ([details](database.md)) | +| `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker ([details](celery-broker.md)) | +| `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](../operating/observability.md)) | +| `--install-dir PATH`, `--instance-name NAME` | Installs elsewhere, or a second copy on the same host ([details](multiple-instances.md)) | +| `--branch BRANCH` | Installs another branch than `main` | + +`install.sh --help` lists every flag, including those for an existing database or broker. + +The installer: + +1. installs the system packages and Rust; +2. picks the service user: the user who ran `sudo`, or a new `relaysms` user when run from a root shell; +3. clones the repository, builds it and creates `.env` with fresh encryption keys; +4. sets up the database and broker you chose, and applies the database migrations; +5. installs and starts the systemd services; +6. sets up nginx and the certificate, if you gave a domain. + +Running it again updates the install in place and keeps its `.env`. Run it as the same user as the first time; it refuses another service user. + +## After installing + +1. Go to the install directory and check that the services are up: + + ```bash + cd /opt/relaysms/relaysms-publisher + sudo ./manage.sh status + curl http://127.0.0.1:16000/health # or https:///health with nginx + ``` + + You should see `{"status":"ok"}`. + +2. Set what you need in `.env` (`template.env` describes each setting), then check it and restart: + + ```bash + sudo ./manage.sh check + sudo ./manage.sh restart + ``` + +3. Create an administrator credential: + + ```bash + sudo ./publisher.sh creds create --username ops --administrator + ``` + + > [!IMPORTANT] + > The password is printed once. Store it before closing the terminal. + +4. Back up `.env`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. + +5. [Add the platform adapters](../platforms/manage.md) you want to offer, and [register the gateway clients](../gateway-clients/manage.md) that relay SMS to this server. + +[Services](../operating/services.md) covers `manage.sh` and `publisher.sh`. + +## Serving the manual + +`--docs-site-name` builds this manual into `book/` and serves it with nginx on its own domain and certificate. `manage.sh update` rebuilds it. + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ + sudo bash -s -- --site-name publisher.example.com --docs-site-name docs.publisher.example.com +``` diff --git a/docs/operating/services.md b/docs/operating/services.md new file mode 100644 index 0000000..9dfa11d --- /dev/null +++ b/docs/operating/services.md @@ -0,0 +1,60 @@ +# Services + +The Publisher runs as five systemd services, grouped under `relaysms-publisher.target`: + +| Service | Runs | Listens on | +| --- | --- | --- | +| `relaysms-publisher-rest` | REST API | `HOST:PORT` (`127.0.0.1:16000`) | +| `relaysms-publisher-grpc` | gRPC API | `GRPC_HOST:GRPC_PORT` (`127.0.0.1:6000`) | +| `relaysms-publisher-worker` | Celery worker: publishes queued payloads | - | +| `relaysms-publisher-beat` | Celery Beat: schedules cleanups, and the worker heartbeat with Uptime Kuma | - | +| `relaysms-publisher-smtp` | [Email listener](../configuration/smtp-transport.md); exits at once while `SMTP_TRANSPORT_ENABLED` is off | - | + +## manage.sh + +Run it from the install directory with `sudo`: + +| Command | Does | +| --- | --- | +| `status` | Shows each service's state | +| `logs` | Follows the service logs | +| `start`, `stop`, `restart` | Controls every service together | +| `enable`, `disable` | Starts the services on boot, or stops doing so | +| `update` | Pulls the latest code, rebuilds and restarts the services | +| `migrate` | Applies database migrations | +| `check` | Reports every missing or invalid setting | +| `nginx [DOMAIN]` | Renders the nginx site again and reattaches or obtains its certificate ([details](../installation/reverse-proxy.md#on-an-existing-install)) | +| `uninstall` | Removes the services and the install directory, with `.env` and `data/`. nginx sites, certificates, database and broker servers, and the service user stay. | + +`logs` takes `-u/--unit` (`rest`, `grpc`, `worker`, `beat` or `smtp`; repeatable), `-n/--lines` and `-s/--since`, and `--no-follow` prints and exits: + +```bash +sudo ./manage.sh logs --unit rest --unit grpc --since "1 hour ago" --lines 200 +``` + +> [!IMPORTANT] +> Run `update --migrate` to apply the release's database migrations; with none pending it changes nothing. They run after the dependencies are reinstalled and before the services restart. + +`update` also checks `.env`. When a setting is invalid, it still restarts every service, then exits with an error; services that use that setting fail to start. + +## publisher.sh + +`./publisher.sh` runs the admin CLI as the service user, with the virtualenv and `.env`: + +```bash +sudo ./publisher.sh --help # every command +sudo ./publisher.sh env # install directory, service user and the .env values the CLI uses +sudo ./publisher.sh shell # a shell as the service user, with .env loaded +``` + +Use it instead of `python3 -m publisher`: run as another user, the CLI can leave adapter files the services can't read. + +## Files + +| Path | Holds | +| --- | --- | +| `/opt/relaysms/relaysms-publisher/` | The install | +| `.env` | Settings (`root:`, mode 640) | +| `data/` | SQLite database, Celery files and adapters (owned by the service user, mode 750) | +| `/etc/systemd/system/relaysms-publisher*` | Service units | +| `/etc/nginx/sites-available/.conf` | nginx sites | diff --git a/docs/platforms/manage.md b/docs/platforms/manage.md index 2e11eed..0fe4d61 100644 --- a/docs/platforms/manage.md +++ b/docs/platforms/manage.md @@ -2,23 +2,22 @@ Each platform (Gmail, Telegram, Mastodon, ...) is served by an adapter installed from its own GitHub repository. Its files live under `data/platforms/` (see [Files](#files)). -> [!IMPORTANT] -> Run these commands with `./publisher.sh` from the install directory, not `python3 -m publisher`. It runs as the service user, so the services can still read the adapter files. +Run these commands from the install directory with [`publisher.sh`](../operating/services.md#publishersh), which runs them as the service user. ## Commands ```bash -./publisher.sh platforms add [--tag TAG] # install the newest, or a given, version tag -./publisher.sh platforms update [NAME] [--tag TAG] # move one adapter, or all, to a version tag -./publisher.sh platforms list # every adapter, disabled ones included -./publisher.sh platforms disable # hide it from users; it still revokes tokens -./publisher.sh platforms enable # offer it again -./publisher.sh platforms remove [--force] # uninstall -./publisher.sh platforms config show # its settings, without secret values -./publisher.sh platforms config set KEY=VALUE|KEY # change settings; a bare KEY asks for the value -./publisher.sh platforms setup [--input KEY=VALUE] # run the adapter's setup and save its settings -./publisher.sh platforms import # register adapter directories, move their files out of the code -./publisher.sh platforms exec -- [ARGS] # run a command the adapter installs +sudo ./publisher.sh platforms add [--tag TAG] # install the newest, or a given, version tag +sudo ./publisher.sh platforms update [NAME] [--tag TAG] # move one adapter, or all, to a version tag +sudo ./publisher.sh platforms list # every adapter, disabled ones included +sudo ./publisher.sh platforms disable # hide it from users; it still revokes tokens +sudo ./publisher.sh platforms enable # offer it again +sudo ./publisher.sh platforms remove [--force] # uninstall +sudo ./publisher.sh platforms config show # its settings, without secret values +sudo ./publisher.sh platforms config set KEY=VALUE|KEY # change settings; a bare KEY asks for the value +sudo ./publisher.sh platforms setup [--input KEY=VALUE] # run the adapter's setup and save its settings +sudo ./publisher.sh platforms import # register adapter directories, move their files out of the code +sudo ./publisher.sh platforms exec -- [ARGS] # run a command the adapter installs ``` Commands taking a name also take `--proto-id` and `--cat-id`, for when a name matches more than one adapter. Administrators can do the same over the [REST API](../reference/rest-api.md#managing-platform-adapters), but only from GitHub orgs listed in `PLATFORMS_GITHUB_ORGS`. @@ -53,15 +52,15 @@ An update builds the new version and its virtualenv beside the running one, then An adapter reads its settings, such as API keys, from `credentials.json` in its config directory. Its `adapter.toml` declares each one. Lists are comma-separated. ```bash -./publisher.sh platforms config set rmail smtp_host=smtp.example.com smtp_port=465 -./publisher.sh platforms config set rmail smtp_password -./publisher.sh platforms config set rmail --unset alias_prefix +sudo ./publisher.sh platforms config set rmail smtp_host=smtp.example.com smtp_port=465 +sudo ./publisher.sh platforms config set rmail smtp_password +sudo ./publisher.sh platforms config set rmail --unset alias_prefix ``` `config set` changes only the keys given. `setup` is for adapters whose settings come from the platform (e.g. a registered app for Mastodon, a downloaded client file for Gmail) and replaces all of them. `--input KEY=@PATH` reads a file: ```bash -./publisher.sh platforms setup gmail --input client_file=@client_secret.json +sudo ./publisher.sh platforms setup gmail --input client_file=@client_secret.json ``` ## An adapter's own commands @@ -69,7 +68,7 @@ An adapter reads its settings, such as API keys, from `credentials.json` in its An adapter may install admin commands. `exec` runs one from the adapter's virtualenv with its config and state directories; put `--` before it: ```bash -./publisher.sh platforms exec -- --help +sudo ./publisher.sh platforms exec -- --help ``` ## Writing an adapter diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 2937ed7..9377f88 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -16,7 +16,7 @@ https:///v1 ## Authentication -Credentials and their scopes are managed with [`./publisher.sh creds`](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#credentials) or the `/v1/creds` endpoints. +Credentials and their scopes are managed with [`sudo ./publisher.sh creds`](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#credentials) or the `/v1/creds` endpoints. * **Web session:** `POST /v1/auth/login`, then send the cookie with every request (`credentials: "include"` in `fetch`). Writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from this API's origin or `AUTH_WEB_ORIGINS`. Sessions end after `AUTH_SESSION_IDLE_MINUTES` idle (30) or `AUTH_SESSION_MAX_HOURS` (12). * **HTTP Basic:** username and password on every request, e.g. `curl -u analyst: .../v1/stats/publications`. From 919ee74c63198a3860ee665475af00aa6e26b478 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Thu, 22 Oct 2026 19:51:34 +0100 Subject: [PATCH 08/13] docs: add the settings reference and offline publishing pages --- README.md | 26 +---- docs/SUMMARY.md | 4 +- docs/configuration/encryption-keys.md | 4 +- docs/configuration/offline-publishing.md | 37 +++++++ docs/configuration/settings.md | 128 +++++++++++++++++++++++ docs/configuration/smtp-transport.md | 19 +--- docs/installation/database.md | 2 +- docs/installation/docker.md | 2 +- docs/installation/manual.md | 2 +- docs/installation/multiple-instances.md | 2 +- docs/installation/server.md | 2 +- docs/openapi.json | 2 +- docs/reference/rest-api.md | 2 +- publisher/api/rest/v1/publications.py | 2 +- template.env | 6 +- tests/test_config.py | 22 +++- 16 files changed, 204 insertions(+), 58 deletions(-) create mode 100644 docs/configuration/offline-publishing.md create mode 100644 docs/configuration/settings.md diff --git a/README.md b/README.md index 0c59b08..bfed4cb 100644 --- a/README.md +++ b/README.md @@ -161,31 +161,7 @@ python -m publisher seed gateway-clients --count 10 # gateway clients, some ## Configuration -Settings live in `.env`. `template.env` lists every setting with its default and what it does, and `python -m publisher config check` (or `sudo ./manage.sh check` on a server) reports any that are missing or invalid. - -| Settings | Documented in | -| --- | --- | -| Database, Celery broker, encryption keys | [Database](docs/installation/database.md), [Celery broker](docs/installation/celery-broker.md), [Encryption keys](docs/configuration/encryption-keys.md) | -| REST API login and sessions (`AUTH_*`) | [REST API](docs/reference/rest-api.md#authentication) | -| SMTP transport (`SMTP_*`) | [SMTP transport](docs/configuration/smtp-transport.md#configuration) | -| Tracing, metrics and logs (`OTEL_*`) | [Observability](docs/operating/observability.md) | -| Offline publishing (`OFFLINE_PUBLISH_*`) | [Below](#offline-publishing) | - -### Offline Publishing - -Offline payloads are tagged with the protocol they arrived on: `https` for [REST `/publications`](docs/reference/rest-api.md#publishing) and `smtp` for the [SMTP transport](docs/configuration/smtp-transport.md). - -```bash -OFFLINE_PUBLISH_ALLOWED_PROTOCOLS=smtp # comma-separated; empty allows every protocol -OFFLINE_PUBLISH_SHARED_SECRET= # 64-char hex; required when https is allowed -``` - -- Offline payloads from a protocol not in `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` are discarded. -- Offline payloads over `https` must carry `OFFLINE_PUBLISH_SHARED_SECRET` in the request's `tag` field. -- Allowing `https`, or every protocol, without the secret fails `config check` and startup. - -> [!TIP] -> For a Twilio number, run [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) with `FORWARD_URLS_JSON` set to this server's `/v1/publications` URL and `FORWARD_TAG` set to `OFFLINE_PUBLISH_SHARED_SECRET`, then allow `https`. +Settings live in `.env`. The [settings reference](docs/configuration/settings.md) lists each one, and `sudo ./manage.sh check` reports any that are missing or invalid. ## Platform adapters diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 4741dc6..ac48813 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -19,9 +19,9 @@ # Configuration -- [Settings reference]() +- [Settings reference](configuration/settings.md) - [Encryption keys](configuration/encryption-keys.md) -- [Offline publishing]() +- [Offline publishing](configuration/offline-publishing.md) - [SMTP transport](configuration/smtp-transport.md) # Platform adapters diff --git a/docs/configuration/encryption-keys.md b/docs/configuration/encryption-keys.md index d6d8da1..631091c 100644 --- a/docs/configuration/encryption-keys.md +++ b/docs/configuration/encryption-keys.md @@ -5,8 +5,8 @@ Three keys in `.env` protect the data. The installer and `make .env` generate th | Setting | Protects | Used | | --- | --- | --- | | `DATA_ENCRYPTION_KEY` | The server's private keys, and the keyed hash that replaces each sender's address | Always | -| `DATABASE_FIELD_ENCRYPTION_KEY` | Linked accounts' platform tokens | When `DATABASE_FIELD_ENCRYPTION_ENABLED=true` (the default) | -| `DATABASE_ENCRYPTION_KEY` | The whole SQLite database file (SQLCipher) | When `DATABASE_ENCRYPTION_ENABLED=true` (the default) and the database is SQLite | +| `DATABASE_FIELD_ENCRYPTION_KEY` | Linked accounts' platform tokens | When `DATABASE_FIELD_ENCRYPTION_ENABLED=true`, as `template.env` sets it | +| `DATABASE_ENCRYPTION_KEY` | The whole SQLite database file (SQLCipher) | When `DATABASE_ENCRYPTION_ENABLED=true`, as `template.env` sets it, and the database is SQLite | > [!CAUTION] > Back up `.env` with the database, and never change a key once the install holds data. Data encrypted with a lost or changed key can't be read again, and a new `DATA_ENCRYPTION_KEY` also gives every sender a new hash. diff --git a/docs/configuration/offline-publishing.md b/docs/configuration/offline-publishing.md new file mode 100644 index 0000000..f15de06 --- /dev/null +++ b/docs/configuration/offline-publishing.md @@ -0,0 +1,37 @@ +# Offline publishing + +An offline payload carries no linked account: the app encrypts it to the server's identity key, and the Publisher publishes it through the RelaySMS Mail adapter (`rmail`), which must be installed. Anyone who can reach an allowed protocol can send one. + +Each payload is tagged with the protocol it arrived on: + +| Protocol | Arrives through | +| --- | --- | +| `https` | [REST `POST /v1/publications`](../reference/rest-api.md#publishing) | +| `smtp` | The [SMTP transport](smtp-transport.md) | + +Settings: [Offline publishing](settings.md#offline-publishing). + +- An offline payload from a protocol not in `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` is discarded. +- An offline payload over `https` must carry `OFFLINE_PUBLISH_SHARED_SECRET` in the request's `tag` field, or it's discarded. +- Leaving `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` empty, or listing `https`, without the secret fails `manage.sh check`, and the services refuse to start. +- A discarded payload is logged and recorded as `failed` in the publication stats, with reason `protocol_not_allowed`, `offline_tag_missing` or `offline_tag_invalid`. `POST /v1/publications` still returns success, since the check runs in the queue. + +## Allowing https + +1. Generate a secret: + + ```bash + openssl rand -hex 32 + ``` + +2. Set `OFFLINE_PUBLISH_SHARED_SECRET` to it, and add `https` to `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS`, e.g. `smtp,https`. +3. Check and restart: + + ```bash + sudo ./manage.sh check + sudo ./manage.sh restart + ``` + +4. Give the secret to each gateway client that posts to `POST /v1/publications`, to send as `tag`. + +For a Twilio number, [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) can post to this server: set its `FORWARD_URLS_JSON` to `https:///v1/publications` and `FORWARD_TAG` to the secret. diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md new file mode 100644 index 0000000..4a28623 --- /dev/null +++ b/docs/configuration/settings.md @@ -0,0 +1,128 @@ +# Settings reference + +Settings live in `.env` in the install directory. `template.env` lists each one with its default. + +- `sudo ./manage.sh check` reports missing or invalid settings; each service refuses to start on an error in the settings it uses. +- Restart the services after a change: `sudo ./manage.sh restart`. +- After an update, `sudo ./sync-env.sh` adds the settings from `template.env` that `.env` is missing, and shows the change before writing it. +- On Docker, check with `docker compose exec publisher python -m publisher config check` and apply a change with `docker compose up -d`. + +## Server + +| Setting | Default | Description | +| --- | --- | --- | +| `HOST` | `127.0.0.1` | REST API address. Docker always binds `0.0.0.0`. | +| `PORT` | `16000` | REST API port | +| `WORKERS` | - | REST API worker processes: `1` by default under `scripts/run.sh`, `4` on Docker. The server install always runs `4`. | +| `GRPC_HOST` | `127.0.0.1` | gRPC address. Docker always binds `0.0.0.0`. | +| `GRPC_PORT` | `6000` | gRPC port | +| `GRPC_MAX_WORKERS` | `10` | gRPC worker threads | +| `GRPC_TLS_ENABLED` | `false` | Serves TLS on `GRPC_PORT` itself; leave off behind nginx ([details](../installation/reverse-proxy.md#grpc-tls-without-nginx)) | +| `GRPC_TLS_CERT_FILE`, `GRPC_TLS_KEY_FILE` | - | Certificate and key. Required when `GRPC_TLS_ENABLED=true`. | +| `NONCE_TTL_SECONDS` | `600` | How far a gRPC request's timestamp may be from the server's clock; nonces are remembered this long to refuse replays | + +## Database + +See [Database](../installation/database.md). Keys are 64 hex characters; see [Encryption keys](encryption-keys.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `DATABASE_DIALECT` | `sqlite` | `sqlite`, `mysql` (MySQL and MariaDB) or `postgres` | +| `SQLITE_DATABASE_PATH` | `data/relaysms.db` | SQLite database file | +| `MYSQL_HOST`, `MYSQL_PORT` | `127.0.0.1`, `3306` | MySQL server | +| `MYSQL_USER`, `MYSQL_PASSWORD` | - | Required for `mysql` | +| `MYSQL_DATABASE` | `relaysms` | Required for `mysql`; created if it doesn't exist | +| `POSTGRES_HOST`, `POSTGRES_PORT` | `127.0.0.1`, `5432` | PostgreSQL server | +| `POSTGRES_USER`, `POSTGRES_PASSWORD` | - | Required for `postgres` | +| `POSTGRES_DATABASE` | `relaysms` | Required for `postgres`; created if it doesn't exist | +| `DATABASE_ENCRYPTION_ENABLED` | `true` (`false` if unset) | Encrypts the SQLite file with SQLCipher. No effect on MySQL or PostgreSQL. | +| `DATABASE_ENCRYPTION_KEY` | - | Required for SQLite when encryption is on | +| `DATABASE_FIELD_ENCRYPTION_ENABLED` | `true` (`false` if unset) | Encrypts linked accounts' platform tokens | +| `DATABASE_FIELD_ENCRYPTION_KEY` | - | Required when field encryption is on | +| `DATA_ENCRYPTION_KEY` | - | Required. Encrypts the server's private keys, and keys the hash that replaces each sender's address. | + +## Celery + +See [Celery broker](../installation/celery-broker.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `CELERY_BROKER_TYPE` | `sqlite` | `sqlite`, `redis` or `rabbitmq` | +| `CELERY_WORKER_CONCURRENCY` | `1` for `sqlite`, else `4` | Tasks the worker runs at once | +| `CELERY_BROKER_DB_PATH`, `CELERY_RESULT_DB_PATH` | `data/celery_broker.db`, `data/celery_results.db` | Broker and result files when `CELERY_BROKER_TYPE=sqlite` | +| `CELERY_REDIS_URL` | `redis://localhost:6379/0` | Broker and result backend when `CELERY_BROKER_TYPE=redis` | +| `CELERY_RABBITMQ_URL` | `amqp://guest:guest@localhost:5672//` | Broker when `CELERY_BROKER_TYPE=rabbitmq` | +| `CELERY_BEAT_SCHEDULE_PATH` | `data/celerybeat-schedule` | Beat's schedule file | +| `CELERY_CLEANUP_CRON` | `0 */3 * * *` | Cron schedule (5 fields, UTC) for deleting incomplete payloads, ended web sessions, and audit events and adapter jobs past retention, and failing abandoned adapter jobs | +| `CELERY_TOKEN_CLEANUP_CRON` | `0 3 * * *` | Cron schedule (5 fields, UTC) for deleting tokens idle past `TOKEN_IDLE_MAX_AGE_DAYS` and revoking them on their platform | + +## Retention + +| Setting | Default | Description | +| --- | --- | --- | +| `PAYLOAD_SESSION_MAX_AGE_HOURS` | `3` | Hours before an incomplete multi-part payload is discarded | +| `TOKEN_IDLE_MAX_AGE_DAYS` | `90` | Days without use before a linked account's token is deleted and revoked on its platform | +| `AUDIT_RETENTION_DAYS` | `365` | Days to keep audit events and adapter install and update jobs | + +## Offline publishing + +See [Offline publishing](offline-publishing.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` | `smtp` (empty if unset) | Comma-separated protocols whose offline payloads publish: `https`, `smtp`. Empty allows both. | +| `OFFLINE_PUBLISH_SHARED_SECRET` | - | 64 hex characters. Required when `https` is allowed. | + +## SMTP transport + +See [SMTP transport](smtp-transport.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `SMTP_TRANSPORT_ENABLED` | `false` | Runs the email listener | +| `SMTP_IMAP_SERVER`, `SMTP_IMAP_USERNAME`, `SMTP_IMAP_PASSWORD` | - | IMAP host and login. Required when the transport is enabled. | +| `SMTP_IMAP_PORT` | `993` | IMAP port | +| `SMTP_IMAP_MAIL_FOLDER` | `INBOX` | Comma-separated folders to watch, e.g. `INBOX,Spam` | +| `SMTP_TLS_CLIENT_CERTIFICATE`, `SMTP_TLS_CLIENT_KEY` | - | mTLS client certificate and key, if the provider requires one. Set both or neither. | +| `SMTP_ALLOWED_SENDERS` | - | Comma-separated addresses and domains allowed to send. Empty allows nothing. | +| `SMTP_TRUSTED_AUTHSERV_ID` | - | `authserv-id` whose `Authentication-Results` verdicts are trusted, e.g. `mx.google.com`. Required when the transport is enabled. | +| `SMTP_REQUIRE_DKIM`, `SMTP_REQUIRE_SPF` | `true` | Require `dkim=pass` and `spf=pass` | +| `SMTP_VERIFY_DKIM_INDEPENDENTLY` | `false` | Also re-verifies the DKIM signature against DNS | + +## Platform adapters + +See [Managing adapters](../platforms/manage.md#files). Relative paths are from the install directory. + +| Setting | Default | Description | +| --- | --- | --- | +| `PLATFORMS_ADAPTERS_DIR` | `data/platforms/adapters` | Directory for adapters' code | +| `PLATFORMS_ADAPTERS_VENV_DIR` | `data/platforms/venvs` | Directory for adapters' virtualenvs | +| `PLATFORMS_ADAPTERS_CONFIG_DIR` | `data/platforms/config` | Directory for adapters' `credentials.json` files | +| `PLATFORMS_ADAPTERS_STATE_DIR` | `data/platforms/state` | Directory for files adapters write, such as databases | +| `PLATFORMS_GITHUB_ORGS` | - | Comma-separated GitHub orgs administrators may install and update adapters from over the REST API. Empty refuses both; `publisher.sh platforms` is unaffected. | + +## REST API access + +See [REST API](../reference/rest-api.md#authentication). + +| Setting | Default | Description | +| --- | --- | --- | +| `AUTH_WEB_ORIGINS` | - | Comma-separated web client origins allowed to call the API from a browser with a session cookie, e.g. `https://console.example.org`. Must be the same site as the API. Empty when the web client is served from the API's own origin. | +| `AUTH_SESSION_COOKIE_SECURE` | `true` | Sends the session cookie over HTTPS only. Set `false` only for local HTTP development. | +| `AUTH_SESSION_IDLE_MINUTES` | `30` | Minutes of inactivity that end a web session | +| `AUTH_SESSION_MAX_HOURS` | `12` | Hours after login when a web session ends, even if it's active | +| `API_DOCS_ENABLED` | `false` | Serves interactive API docs at `/docs` and the schema at `/openapi.json`. For development. | + +## Logging and monitoring + +See [Observability](../operating/observability.md). + +| Setting | Default | Description | +| --- | --- | --- | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warning`, `error` or `critical` | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | - | OTLP collector URL for traces, metrics and logs, e.g. `http://localhost:4317`. Needs `requirements-observability.txt`. Blank turns all three off. | +| `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | `grpc` or `http/protobuf` | +| `OTEL_TRACES_SAMPLER`, `OTEL_TRACES_SAMPLER_ARG` | `parentbased_traceidratio`, `1.0` | Which traces to keep, and the fraction kept (`0` to `1`) | +| `OTEL_LOGS_EXPORTER` | `otlp` | `otlp` sends logs to the collector; `none` doesn't | +| `OTEL_PYTHON_LOG_CORRELATION` | `true` | Adds trace and span IDs to log records | +| `UPTIME_KUMA_WORKER_PUSH_URL`, `UPTIME_KUMA_SMTP_PUSH_URL` | - | Uptime Kuma push URLs for the worker and email listener heartbeats. Blank turns them off. | diff --git a/docs/configuration/smtp-transport.md b/docs/configuration/smtp-transport.md index f7319ed..6bd8b76 100644 --- a/docs/configuration/smtp-transport.md +++ b/docs/configuration/smtp-transport.md @@ -41,22 +41,9 @@ Two checks must both pass before a message is queued: ## Configuration -| Variable | Default | Description | -| :--- | :--- | :--- | -| `SMTP_TRANSPORT_ENABLED` | `false` | Enables the listener. | -| `SMTP_IMAP_SERVER` | - | IMAP host. Required when enabled. | -| `SMTP_IMAP_PORT` | `993` | IMAP port. | -| `SMTP_IMAP_USERNAME` | - | Required when enabled. | -| `SMTP_IMAP_PASSWORD` | - | Required when enabled. | -| `SMTP_IMAP_MAIL_FOLDER` | `INBOX` | Comma-separated folders to watch. | -| `SMTP_TLS_CLIENT_CERTIFICATE` / `SMTP_TLS_CLIENT_KEY` | - | Optional mTLS client certificate and key, if the provider requires one. Set both or neither. | -| `SMTP_ALLOWED_SENDERS` | - | Comma-separated allowlist of addresses and/or domains. | -| `SMTP_TRUSTED_AUTHSERV_ID` | - | `authserv-id` whose `Authentication-Results` verdicts are trusted. Required when enabled. | -| `SMTP_REQUIRE_DKIM` | `true` | Require `dkim=pass`. | -| `SMTP_REQUIRE_SPF` | `true` | Require `spf=pass`. | -| `SMTP_VERIFY_DKIM_INDEPENDENTLY` | `false` | Also re-verify the DKIM signature via DNS. | - -The listener tags what it queues with protocol `smtp`. Whether offline payloads from email publish depends on [offline publishing](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#offline-publishing). +Settings: [SMTP transport](settings.md#smtp-transport). + +The listener tags what it queues with protocol `smtp`. Whether offline payloads from email publish depends on [offline publishing](offline-publishing.md). ## Running diff --git a/docs/installation/database.md b/docs/installation/database.md index e0a213d..dd7dde8 100644 --- a/docs/installation/database.md +++ b/docs/installation/database.md @@ -10,7 +10,7 @@ Sensitive fields are also encrypted in every dialect; see [Encryption keys](../c ## MySQL, MariaDB or PostgreSQL -A setup script installs the server, creates the database and user, and writes the connection details to `.env`. Running it again without `--db-password` sets a new password. The MySQL script installs `mysql-server`, which only Ubuntu ships; on Debian, set up MariaDB yourself and pass `--db-existing`. +A setup script installs the server, creates the database and user, and writes the connection details to `.env`. Running it again without `--db-password` gives the user a new password; restart the services afterwards. The MySQL script installs `mysql-server`, which only Ubuntu ships; on Debian, set up MariaDB yourself and pass `--db-existing`. ```bash sudo ./install.sh --setup-db postgres # during install diff --git a/docs/installation/docker.md b/docs/installation/docker.md index 8672eaf..7470066 100644 --- a/docs/installation/docker.md +++ b/docs/installation/docker.md @@ -1,6 +1,6 @@ # Install with Docker -`docker-compose.yml` runs every service in one container, with its data in `./data` and its settings in `.env` (`template.env` describes each one): +`docker-compose.yml` runs every service in one container, with its data in `./data` and its settings in `.env` (see the [settings reference](../configuration/settings.md)): ```bash git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git diff --git a/docs/installation/manual.md b/docs/installation/manual.md index ddafca8..24c9e65 100644 --- a/docs/installation/manual.md +++ b/docs/installation/manual.md @@ -41,7 +41,7 @@ These are the steps the [installer](server.md) runs, for when you need to change make build ``` -6. Create `.env` with fresh encryption keys, then set what you need (`template.env` describes each one): +6. Create `.env` with fresh encryption keys, then set what you need (see the [settings reference](../configuration/settings.md)): ```bash make .env diff --git a/docs/installation/multiple-instances.md b/docs/installation/multiple-instances.md index e3fbdcb..dbde213 100644 --- a/docs/installation/multiple-instances.md +++ b/docs/installation/multiple-instances.md @@ -2,7 +2,7 @@ To run a second Publisher on the same host, give it its own install directory, instance name and ports: -1. Install it without nginx. When it shares a RabbitMQ or database server with the first instance, also pass `--broker-vhost acme` or `--db-name acme`: +1. Install it without nginx. When it shares a database or RabbitMQ server with the first instance, also pass `--db-name acme --db-user acme` or `--broker-vhost acme --broker-user acme`; reusing the first instance's user would give it a new password and break the first instance: ```bash sudo ./install.sh --install-dir /srv/relaysms-acme --instance-name acme --skip-nginx diff --git a/docs/installation/server.md b/docs/installation/server.md index e194a69..544880f 100644 --- a/docs/installation/server.md +++ b/docs/installation/server.md @@ -52,7 +52,7 @@ Running it again updates the install in place and keeps its `.env`. Run it as th You should see `{"status":"ok"}`. -2. Set what you need in `.env` (`template.env` describes each setting), then check it and restart: +2. Set what you need in `.env` (see the [settings reference](../configuration/settings.md)), then check it and restart: ```bash sudo ./manage.sh check diff --git a/docs/openapi.json b/docs/openapi.json index 434a82f..2686bb2 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1507,7 +1507,7 @@ "Publishing" ], "summary": "Publish", - "description": "Queues an SMS payload for publishing.\n\nMulti-part payloads are kept until every segment arrives, then published. Payloads\nare tagged with protocol `https`; see Offline Publishing in the README for\n`OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors\nhappen later in the queue, so they're logged, not returned.", + "description": "Queues an SMS payload for publishing.\n\nMulti-part payloads are kept until every segment arrives, then published. Payloads\nare tagged with protocol `https`; see Offline publishing in the manual for\n`OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors\nhappen later in the queue, so they're logged, not returned.", "operationId": "create_publications_v1_publications_post", "requestBody": { "content": { diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 9377f88..6076cb6 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -66,7 +66,7 @@ List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. ## Publishing -`POST /v1/publications` takes JSON and tags payloads with protocol `https`. See [offline publishing](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#offline-publishing) for how the protocol and `tag` decide whether offline payloads publish. +`POST /v1/publications` takes JSON and tags payloads with protocol `https`. See [offline publishing](../configuration/offline-publishing.md) for how the protocol and `tag` decide whether offline payloads publish. It only checks the payload's structure before queueing it. Decryption, payload type and adapter errors happen later, so they're logged on the server, not returned. diff --git a/publisher/api/rest/v1/publications.py b/publisher/api/rest/v1/publications.py index c374304..52f74da 100644 --- a/publisher/api/rest/v1/publications.py +++ b/publisher/api/rest/v1/publications.py @@ -26,7 +26,7 @@ def create_publications(body: PublishRestContentRequest) -> PublishContentRespon """Queues an SMS payload for publishing. Multi-part payloads are kept until every segment arrives, then published. Payloads - are tagged with protocol `https`; see Offline Publishing in the README for + are tagged with protocol `https`; see Offline publishing in the manual for `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` and `tag`. Decryption and adapter errors happen later in the queue, so they're logged, not returned. """ diff --git a/template.env b/template.env index 7711d8c..d8e2de2 100644 --- a/template.env +++ b/template.env @@ -1,7 +1,8 @@ # REST API HOST=127.0.0.1 PORT=16000 -# Worker processes for scripts/run.sh (default 1) and Docker (default 4). +# Worker processes for scripts/run.sh (default 1) and Docker (default 4). The +# systemd unit always runs 4. # WORKERS= # gRPC @@ -95,7 +96,7 @@ OFFLINE_PUBLISH_ALLOWED_PROTOCOLS=smtp OFFLINE_PUBLISH_SHARED_SECRET= # SMTP Ingestion Transport -# Mail body must be JSON: {"address": "", "text": ""} +# Mail body must be JSON: {"address": "", "text": ""} SMTP_TRANSPORT_ENABLED=false SMTP_IMAP_SERVER= SMTP_IMAP_PORT=993 @@ -112,7 +113,6 @@ SMTP_TLS_CLIENT_KEY= SMTP_ALLOWED_SENDERS= # authserv-id trusted for Authentication-Results verdicts (e.g. mx.google.com). # Required when SMTP_TRANSPORT_ENABLED=true -# unset rejects all mail. SMTP_TRUSTED_AUTHSERV_ID= SMTP_REQUIRE_DKIM=true SMTP_REQUIRE_SPF=true diff --git a/tests/test_config.py b/tests/test_config.py index c2337ed..524ecb8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -31,6 +31,11 @@ def errors(**env): return "\n".join(config.check({**BASE_ENV, **env})) +def template_env_names(): + text = (config.ROOT / "template.env").read_text() + return set(re.findall(r"^\s*(?:#\s*)?([A-Za-z_]\w*)=", text, re.M)) + + def test_defaults(): assert load(LoggingConfig).log_level == "INFO" assert not load(GrpcConfig).tls_enabled @@ -224,8 +229,7 @@ def test_template_env_paths_match_defaults(): def test_template_env_lists_every_variable_config_reads(): # sync-env.sh --prune drops what the template lacks, so a variable config # reads must be in it, if only commented out. - text = (config.ROOT / "template.env").read_text() - template = set(re.findall(r"^\s*(?:#\s*)?([A-Za-z_]\w*)=", text, re.M)) + template = template_env_names() # An empty environment that records every name config asks for. env = Mock() env.get.side_effect = lambda key, default=None: default @@ -242,6 +246,20 @@ def test_template_env_lists_every_variable_config_reads(): assert unused == set(), "in template.env but unused" +def test_settings_reference_matches_template_env(): + template = template_env_names() + reference = (config.ROOT / "docs" / "configuration" / "settings.md").read_text() + # Only the first column, so a name mentioned in a description doesn't count. + documented = { + name + for row in re.findall(r"^\|\s*(`[A-Z].*?)\s*\|", reference, re.M) + for name in re.findall(r"`([A-Z][A-Z0-9_]*)`", row) + } + + assert template - documented == set(), "missing from the settings reference" + assert documented - template == set(), "in the settings reference, not template.env" + + def test_auth_session_defaults(): auth = load(AuthConfig) From 0d48f68a12e015bd54096a20a76989d97fc3b955 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Fri, 23 Oct 2026 00:30:12 +0100 Subject: [PATCH 09/13] feat(manage): add manage.sh docs to serve the manual on a running install --- install.sh | 5 ---- manage.sh | 77 +++++++++++++++++++++++++++++++++++++++++--------- scripts/lib.sh | 6 ++++ 3 files changed, 69 insertions(+), 19 deletions(-) diff --git a/install.sh b/install.sh index c6300b9..22c8a81 100755 --- a/install.sh +++ b/install.sh @@ -593,11 +593,6 @@ configure_nginx() { install_site "$site" render_nginx_site "$interactive" } -render_docs_site() { - sed -e "s/__SERVER_NAME__/$1/g" -e "s|__BOOK_DIR__|$INSTALL_DIR/book|g" \ - "$INSTALL_DIR/deploy/nginx/relaysms-publisher-docs-nginx.conf.template" >"$2" -} - configure_docs_site() { if [ "${SKIP_NGINX:-0}" = "1" ]; then return diff --git a/manage.sh b/manage.sh index 2ddbcbc..f4e0a95 100755 --- a/manage.sh +++ b/manage.sh @@ -271,6 +271,17 @@ restored on failure. EOF } +docs_usage() { + cat <<'EOF' +Usage: manage.sh docs [DOMAIN] + +Builds the manual into book/, then renders its nginx site from the template, +reattaches or obtains its certificate and reloads nginx. DOMAIN defaults to +this install's manual site. The previous file is kept as .conf.bak and +restored on failure. manage.sh update rebuilds the manual afterwards. +EOF +} + # Matches publisher_rest_ (this install) or an unsuffixed publisher_rest. detect_nginx_site() { local port sites=() @@ -282,9 +293,18 @@ detect_nginx_site() { basename "${sites[0]}" .conf } +detect_docs_site() { + local sites=() + mapfile -t sites < <(grep -lF "root $INSTALL_DIR/book;" /etc/nginx/sites-available/*.conf 2>/dev/null) + [ "${#sites[@]}" -eq 1 ] || + error "Found ${#sites[@]} matching nginx sites; pass the domain: $0 docs DOMAIN" + basename "${sites[0]}" .conf +} + +# $1=domain $2=site file $3=function that writes it, called as "$3 DOMAIN FILE" install_nginx_site() { - local site="$1" conf="$2" - render_nginx_site "$site" "$conf" || return 1 + local site="$1" conf="$2" render="$3" + "$render" "$site" "$conf" || return 1 ln -sf "$conf" "/etc/nginx/sites-enabled/${site}.conf" || return 1 # Re-adds the 443 block that re-rendering dropped. @@ -298,26 +318,21 @@ install_nginx_site() { nginx -t || return 1 } -cmd_nginx() { - case "${1:-}" in - -h | --help) - nginx_usage - return - ;; - esac - require_root +require_nginx() { if ! command -v nginx &>/dev/null || ! command -v certbot &>/dev/null; then error "nginx and certbot must be installed" fi +} - local site="${1:-}" - [ -n "$site" ] || site=$(detect_nginx_site) +# $1=domain $2=function that writes its site file +apply_nginx_site() { + local site="$1" render="$2" validate_hostname "DOMAIN" "$site" local conf="/etc/nginx/sites-available/${site}.conf" [ -f "$conf" ] && cp -p "$conf" "$conf.bak" - if ! install_nginx_site "$site" "$conf"; then + if ! install_nginx_site "$site" "$conf" "$render"; then if [ -f "$conf.bak" ]; then cp -p "$conf.bak" "$conf" nginx -t && systemctl reload nginx @@ -328,6 +343,36 @@ cmd_nginx() { log "nginx site $site updated" } +cmd_nginx() { + case "${1:-}" in + -h | --help) + nginx_usage + return + ;; + esac + require_root + require_nginx + local site="${1:-}" + [ -n "$site" ] || site=$(detect_nginx_site) + apply_nginx_site "$site" render_nginx_site +} + +cmd_docs() { + case "${1:-}" in + -h | --help) + docs_usage + return + ;; + esac + require_root + require_nginx + local site="${1:-}" + [ -n "$site" ] || site=$(detect_docs_site) + validate_hostname "DOMAIN" "$site" + "$INSTALL_DIR/scripts/build-book.sh" + apply_nginx_site "$site" render_docs_site +} + cmd_uninstall() { require_root local confirm @@ -357,7 +402,7 @@ cmd_uninstall() { } usage() { - echo "Usage: $0 {start|stop|restart|status|logs|enable|disable|check|migrate|update|nginx|uninstall}" + echo "Usage: $0 {start|stop|restart|status|logs|enable|disable|check|migrate|update|nginx|docs|uninstall}" exit 1 } @@ -379,6 +424,10 @@ main() { shift cmd_nginx "$@" ;; + docs) + shift + cmd_docs "$@" + ;; update) shift cmd_update "$@" diff --git a/scripts/lib.sh b/scripts/lib.sh index b31fd00..1690217 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -257,6 +257,12 @@ render_nginx_site() { "$INSTALL_DIR/deploy/nginx/relaysms-publisher-nginx.conf.template" >"$conf" } +# Renders the manual's nginx site for $1 into $2, serving $INSTALL_DIR/book. +render_docs_site() { + sed -e "s/__SERVER_NAME__/$1/g" -e "s|__BOOK_DIR__|$INSTALL_DIR/book|g" \ + "$INSTALL_DIR/deploy/nginx/relaysms-publisher-docs-nginx.conf.template" >"$2" +} + install_nginx_certbot() { if ! command -v nginx &>/dev/null || ! command -v certbot &>/dev/null; then log "Installing nginx and certbot" From 99bbba0acf08fae16d2f798ff29c9defd3e8f15c Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Fri, 23 Oct 2026 00:31:21 +0100 Subject: [PATCH 10/13] docs: add provider guides, gateway relays and readability fixes to the manual --- .github/workflows/docs.yml | 4 +- CONTRIBUTING.md | 14 +- README.md | 35 ++-- SECURITY.md | 9 + book.toml | 3 +- docs/README.md | 26 ++- docs/SUMMARY.md | 12 +- docs/configuration/encryption-keys.md | 6 +- docs/configuration/offline-publishing.md | 10 +- docs/configuration/settings.md | 63 ++++--- docs/configuration/smtp-transport.md | 69 ++++--- docs/developers/reference-client.md | 15 +- docs/gateway-clients/connect-a-relay.md | 36 ++++ docs/gateway-clients/manage.md | 31 ++-- docs/getting-started/quick-start.md | 31 +++- docs/installation/celery-broker.md | 21 ++- docs/installation/database.md | 48 +++-- docs/installation/docker.md | 6 +- docs/installation/requirements.md | 18 +- docs/installation/reverse-proxy.md | 9 +- docs/installation/server.md | 47 +++-- docs/operating/observability.md | 225 +++++++++++++---------- docs/operating/services.md | 45 ++++- docs/platforms/bluesky.md | 67 +++++++ docs/platforms/gmail.md | 99 ++++++++++ docs/platforms/manage.md | 74 +++++--- docs/platforms/mastodon.md | 59 ++++++ docs/platforms/relaysms-mail.md | 92 +++++++++ docs/platforms/telegram.md | 49 +++++ docs/reference/grpc-api.md | 26 +-- docs/reference/rest-api.md | 8 +- docs/theme/custom.css | 4 + 32 files changed, 941 insertions(+), 320 deletions(-) create mode 100644 SECURITY.md create mode 100644 docs/gateway-clients/connect-a-relay.md create mode 100644 docs/platforms/bluesky.md create mode 100644 docs/platforms/gmail.md create mode 100644 docs/platforms/mastodon.md create mode 100644 docs/platforms/relaysms-mail.md create mode 100644 docs/platforms/telegram.md create mode 100644 docs/theme/custom.css diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5211792..c3372fa 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,7 +28,7 @@ jobs: - name: Check Markdown links uses: lycheeverse/lychee-action@v2 with: - args: --offline --include-fragments --no-progress README.md CONTRIBUTING.md sdk/README.md "docs/**/*.md" + args: --offline --include-fragments --no-progress README.md CONTRIBUTING.md SECURITY.md sdk/README.md "docs/**/*.md" - name: Build run: make book @@ -37,4 +37,4 @@ jobs: - name: Check book links uses: lycheeverse/lychee-action@v2 with: - args: --offline --include-fragments --no-progress --exclude-path book/404.html "book/**/*.html" + args: --offline --include-fragments --no-progress --root-dir "${{ github.workspace }}/book" "book/**/*.html" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ba0ddfc..6f583ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ ## Reporting security issues > [!CAUTION] -> Don't open a public issue for a vulnerability. Email **** with a description, the affected versions and steps to reproduce. +> Don't open a public issue for a vulnerability. See [SECURITY.md](SECURITY.md). ## Development setup @@ -175,8 +175,8 @@ Common types are `feat`, `fix`, `refactor`, `test`, `docs`, `build`, `ci` and `c - [ ] The PR targets `staging` and does one thing. - [ ] `make check` and `make test` pass locally. -- [ ] New behaviour and bug fixes come with tests. A bug fix has a test that fails without it. -- [ ] Docs, `template.env` and the README are updated if behaviour, settings or commands changed. +- [ ] New behavior and bug fixes come with tests. A bug fix has a test that fails without it. +- [ ] Docs, `template.env` and the README are updated if behavior, settings or commands changed. - [ ] Schema changes include a migration, and a commit message with `[migrate]`. - [ ] The description says what changed, why, and how it was tested. Link the issue with `Closes #123`. - [ ] Anything operators must do on deploy is called out. @@ -215,7 +215,7 @@ The gRPC service handles the account side: storing OAuth2 and PNBA tokens, revok The hooks enforce formatting, lint, types and commit messages. These rules are checked in review: -- **Functions by default.** Modules group related behaviour. Use a class only for real state (a cache, a connection) or when a framework needs one (ORM models, structs, interceptors, servicers, config sections). +- **Functions by default.** Modules group related behavior. Use a class only for real state (a cache, a connection) or when a framework needs one (ORM models, structs, interceptors, servicers, config sections). - **Dependencies are arguments.** Domain and model functions take the database `session` first, then anything else they need, such as the acting credential. - **The entry point owns the transaction.** A route, gRPC handler, task or CLI command opens the session; code below it never opens a session or commits. - **No transaction stays open while an adapter runs,** since a call can take a minute. Publishing is the one exception (see above). @@ -253,7 +253,7 @@ Tests run against in-memory SQLite. `tests/conftest.py` sets the environment bef ### Writing tests - Put a test where its module lives: `publisher/api/rest/v1/stats.py` is tested in `tests/api/rest/test_stats.py`. -- Name tests after the behaviour, such as `test_list_skips_disabled_items`, not the function. +- Name tests after the behavior, such as `test_list_skips_disabled_items`, not the function. - Test through the public interface (a route, an RPC, a domain function). Patch only what crosses the process boundary, such as adapter IPC or outbound HTTP. ### Fixtures and helpers @@ -340,7 +340,9 @@ The manual in `docs/` is an [mdBook](https://rust-lang.github.io/mdBook/) that r - List a new page in `docs/SUMMARY.md`, or the book leaves it out. - Link pages in `docs/` relatively, and files outside it by their full GitHub URL, since the book only has `docs/`. - Draw diagrams as `mermaid` code blocks. -- Write plainly and briefly, one paragraph per line, with sentence-case headings, code blocks for commands and tables for options. +- Start a task page with its requirements, and end it with a link to the next step when that isn't the following page. +- Use tables for reference data, numbered steps for procedures, and one heading per symptom for troubleshooting. +- Write plainly and briefly in US English, one paragraph per line, with sentence-case headings and `` placeholders. ## When CI fails diff --git a/README.md b/README.md index bfed4cb..4bd9348 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The installer sets up everything else, including the system packages and Rust. curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | sudo bash ``` - With no flags it asks about each option. To choose up front, pass flags after `bash -s --`: + It asks about each option you don't pass as a flag. To choose up front, pass flags after `bash -s --`: ```bash curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ @@ -68,22 +68,22 @@ The installer sets up everything else, including the system packages and Rust. 3. Create an administrator credential for the REST API: ```bash - ./publisher.sh creds create --username ops --administrator + sudo ./publisher.sh creds create --username ops --administrator ``` > [!IMPORTANT] - > The password is printed once. Store it before closing the terminal. + > The password is printed only once. -4. Add the platform adapters you want to offer, then follow each adapter's README to configure it: +4. Add the platform adapters you want to offer, then configure each with its [provider guide](docs/platforms/manage.md#provider-guides): ```bash - ./publisher.sh platforms add https://github.com/smswithoutborders/gmail-oauth2-adapter + sudo ./publisher.sh platforms add https://github.com/smswithoutborders/gmail-oauth2-adapter ``` 5. Register the gateway clients that relay SMS to this server: ```bash - ./publisher.sh gateway-clients create --msisdn +237123456789 --protocols https + sudo ./publisher.sh gateway-clients create --msisdn +237123456789 --protocols https ``` 6. Check that the services are up: @@ -109,6 +109,7 @@ From the install directory: | `sudo ./manage.sh migrate` | Apply database migrations | | `sudo ./manage.sh check` | Report every missing or invalid setting | | `sudo ./manage.sh nginx [DOMAIN]` | Re-render the nginx site and reattach or obtain its certificate | +| `sudo ./manage.sh docs [DOMAIN]` | Build the manual and serve it on its own domain | | `sudo ./manage.sh uninstall` | Remove the services and the install directory | See [Services](docs/operating/services.md) for log filters and what `update` and `nginx` do. @@ -174,7 +175,7 @@ Each platform is served by an adapter installed from its own repository. `GET /v - [Bluesky](https://github.com/smswithoutborders/bluesky-oauth2-adapter) - [Mastodon](https://github.com/smswithoutborders/mastodon-oauth2-adapter) -[docs/platforms/manage.md](docs/platforms/manage.md) covers adding, updating, disabling and removing adapters. Each adapter's README covers its own settings. +[Managing adapters](docs/platforms/manage.md) covers adding, updating, disabling and removing adapters, and links a guide for each. ## Gateway clients @@ -185,16 +186,16 @@ Gateway clients are the phone numbers that relay SMS to this server. `GET /v1/ga Credentials log in to the REST API, each with its own scopes. One holding every scope is an administrator. They can also be managed over the [REST API](docs/reference/rest-api.md#managing-credentials). ```bash -./publisher.sh creds scopes # list scopes -./publisher.sh creds create --username ops --administrator # prints the password once -./publisher.sh creds create --username analyst --scope stats:publications:read -./publisher.sh creds set-scopes --username analyst --scope stats:publications:read --scope stats:publications:reasons -./publisher.sh creds list -./publisher.sh creds reset-password --username analyst -./publisher.sh creds disable --username analyst -./publisher.sh creds enable --username analyst -./publisher.sh creds revoke-sessions --username analyst -./publisher.sh creds delete --username analyst +sudo ./publisher.sh creds scopes # list scopes +sudo ./publisher.sh creds create --username ops --administrator # prints the password once +sudo ./publisher.sh creds create --username analyst --scope stats:publications:read +sudo ./publisher.sh creds set-scopes --username analyst --scope stats:publications:read --scope stats:publications:reasons +sudo ./publisher.sh creds list +sudo ./publisher.sh creds reset-password --username analyst +sudo ./publisher.sh creds disable --username analyst +sudo ./publisher.sh creds enable --username analyst +sudo ./publisher.sh creds revoke-sessions --username analyst +sudo ./publisher.sh creds delete --username analyst ``` ## Documentation diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..6334e0e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Security policy + +Don't open a public issue for a vulnerability. Email **** with: + +- a description of the issue and its impact; +- the affected versions or commits; +- steps to reproduce it. + +This covers the Publisher, its adapter SDK in `sdk/` and the platform adapters in the [smswithoutborders](https://github.com/smswithoutborders) organization. diff --git a/book.toml b/book.toml index 5806840..a3e9c80 100644 --- a/book.toml +++ b/book.toml @@ -14,6 +14,7 @@ command = "mdbook-mermaid" [output.html] git-repository-url = "https://github.com/smswithoutborders/RelaySMS-Publisher" -edit-url-template = "https://github.com/smswithoutborders/RelaySMS-Publisher/edit/main/{path}" +edit-url-template = "https://github.com/smswithoutborders/RelaySMS-Publisher/edit/staging/{path}" additional-js = ["mermaid.min.js", "mermaid-init.js"] +additional-css = ["docs/theme/custom.css"] no-section-label = true diff --git a/docs/README.md b/docs/README.md index 5b027f0..6124703 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,13 +5,17 @@ The Publisher posts to online platforms (Gmail, Telegram, Mastodon and more) fro ## How it works ```mermaid +--- +config: + flowchart: + curve: linear +--- flowchart TB app["RelaySMS app"] gateway["Gateway client"] - mailbox["Mailbox"] platform["Gmail, Telegram, ..."] - subgraph publisher["Publisher"] + subgraph publisher[" "] grpc["gRPC API"] rest["REST API"] smtp["Email listener"] @@ -22,8 +26,7 @@ flowchart TB app -- "link an account, online" --> grpc app -- "encrypted SMS" --> gateway gateway -- "HTTPS" --> rest - gateway -- "email" --> mailbox - smtp -- "IMAP" --> mailbox + gateway -- "email, via a mailbox" --> smtp rest --> worker smtp --> worker grpc --> adapter @@ -31,11 +34,21 @@ flowchart TB adapter --> platform ``` +The shaded box is the Publisher. + 1. While online, a user links an account through the gRPC API. The Publisher stores its token encrypted. 2. Offline, the app encrypts the content and sends it by SMS to a gateway client, a phone number that relays SMS. 3. The gateway client posts the payload to the REST API, or emails it to a mailbox the Publisher reads. 4. A Celery worker decrypts the payload and publishes it through the platform's adapter. +## Key terms + +- **Gateway client**: a phone number that relays SMS to the Publisher, from an Android phone running Deku SMS or from Twilio. See [Connecting a relay](gateway-clients/connect-a-relay.md). +- **Platform adapter**: the plugin that publishes to one platform, such as Gmail, installed from its own repository. See [Managing adapters](platforms/manage.md). +- **Linked account**: a user's account on a platform, authorized once while online. The Publisher stores its token encrypted. +- **OAuth2 and PNBA**: the two ways an account links. OAuth2 goes through the platform's consent page (Gmail, Mastodon, Bluesky). Phone number-based authentication (PNBA) uses a code sent to the user's phone (Telegram, RelaySMS Mail). +- **Offline payload**: content sent without a linked account, published through [RelaySMS Mail](platforms/relaysms-mail.md). + ## Where to start | To | Read | @@ -46,3 +59,8 @@ flowchart TB | Relay SMS to the Publisher | [Managing gateway clients](gateway-clients/manage.md) | | Build a client | [gRPC API](reference/grpc-api.md) and [REST API](reference/rest-api.md) | | Understand the protocol | [RelaySMS whitepaper](explanation/relaysms-whitepaper.pdf) | + +## Getting help + +- Ask questions and report bugs in the [issue tracker](https://github.com/smswithoutborders/RelaySMS-Publisher/issues); [CONTRIBUTING.md](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/CONTRIBUTING.md#ways-to-contribute) says what a useful report has. +- Report vulnerabilities privately, as [SECURITY.md](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/SECURITY.md) describes. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index ac48813..5f4e49e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -27,16 +27,16 @@ # Platform adapters - [Managing adapters](platforms/manage.md) -- [Gmail]() -- [Mastodon]() -- [Bluesky]() -- [Telegram]() -- [RelaySMS Mail]() +- [Gmail](platforms/gmail.md) +- [Mastodon](platforms/mastodon.md) +- [Bluesky](platforms/bluesky.md) +- [Telegram](platforms/telegram.md) +- [RelaySMS Mail](platforms/relaysms-mail.md) # Gateway clients - [Managing gateway clients](gateway-clients/manage.md) -- [Connecting a relay]() +- [Connecting a relay](gateway-clients/connect-a-relay.md) # Operating diff --git a/docs/configuration/encryption-keys.md b/docs/configuration/encryption-keys.md index 631091c..7382a2e 100644 --- a/docs/configuration/encryption-keys.md +++ b/docs/configuration/encryption-keys.md @@ -5,8 +5,10 @@ Three keys in `.env` protect the data. The installer and `make .env` generate th | Setting | Protects | Used | | --- | --- | --- | | `DATA_ENCRYPTION_KEY` | The server's private keys, and the keyed hash that replaces each sender's address | Always | -| `DATABASE_FIELD_ENCRYPTION_KEY` | Linked accounts' platform tokens | When `DATABASE_FIELD_ENCRYPTION_ENABLED=true`, as `template.env` sets it | -| `DATABASE_ENCRYPTION_KEY` | The whole SQLite database file (SQLCipher) | When `DATABASE_ENCRYPTION_ENABLED=true`, as `template.env` sets it, and the database is SQLite | +| `DATABASE_FIELD_ENCRYPTION_KEY` | Linked accounts' platform tokens | When field encryption is on | +| `DATABASE_ENCRYPTION_KEY` | The whole SQLite database file (SQLCipher) | When encryption is on and the database is SQLite | + +`template.env` turns both on with `DATABASE_FIELD_ENCRYPTION_ENABLED=true` and `DATABASE_ENCRYPTION_ENABLED=true`. > [!CAUTION] > Back up `.env` with the database, and never change a key once the install holds data. Data encrypted with a lost or changed key can't be read again, and a new `DATA_ENCRYPTION_KEY` also gives every sender a new hash. diff --git a/docs/configuration/offline-publishing.md b/docs/configuration/offline-publishing.md index f15de06..79e131d 100644 --- a/docs/configuration/offline-publishing.md +++ b/docs/configuration/offline-publishing.md @@ -1,13 +1,11 @@ # Offline publishing -An offline payload carries no linked account: the app encrypts it to the server's identity key, and the Publisher publishes it through the RelaySMS Mail adapter (`rmail`), which must be installed. Anyone who can reach an allowed protocol can send one. +Use this to let people send email through [RelaySMS Mail](../platforms/relaysms-mail.md) without linking an account first. Payloads from linked accounts, such as Gmail ones, need none of these settings. An offline payload carries no linked account: the app encrypts it to the server's identity key, and the Publisher publishes it through the RelaySMS Mail adapter (`rmail`), which must be installed. Anyone who can reach an allowed protocol can send one. Each payload is tagged with the protocol it arrived on: -| Protocol | Arrives through | -| --- | --- | -| `https` | [REST `POST /v1/publications`](../reference/rest-api.md#publishing) | -| `smtp` | The [SMTP transport](smtp-transport.md) | +- `https`: [REST `POST /v1/publications`](../reference/rest-api.md#publishing) +- `smtp`: the [SMTP transport](smtp-transport.md) Settings: [Offline publishing](settings.md#offline-publishing). @@ -24,7 +22,7 @@ Settings: [Offline publishing](settings.md#offline-publishing). openssl rand -hex 32 ``` -2. Set `OFFLINE_PUBLISH_SHARED_SECRET` to it, and add `https` to `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS`, e.g. `smtp,https`. +2. Set `OFFLINE_PUBLISH_SHARED_SECRET` to it, and add `https` to `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS`, such as `smtp,https`. 3. Check and restart: ```bash diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 4a28623..0130f1d 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -13,33 +13,35 @@ Settings live in `.env` in the install directory. `template.env` lists each one | --- | --- | --- | | `HOST` | `127.0.0.1` | REST API address. Docker always binds `0.0.0.0`. | | `PORT` | `16000` | REST API port | -| `WORKERS` | - | REST API worker processes: `1` by default under `scripts/run.sh`, `4` on Docker. The server install always runs `4`. | +| `WORKERS` | None | REST API worker processes: `1` by default under `scripts/run.sh`, `4` on Docker. The server install always runs `4`. | | `GRPC_HOST` | `127.0.0.1` | gRPC address. Docker always binds `0.0.0.0`. | | `GRPC_PORT` | `6000` | gRPC port | | `GRPC_MAX_WORKERS` | `10` | gRPC worker threads | | `GRPC_TLS_ENABLED` | `false` | Serves TLS on `GRPC_PORT` itself; leave off behind nginx ([details](../installation/reverse-proxy.md#grpc-tls-without-nginx)) | -| `GRPC_TLS_CERT_FILE`, `GRPC_TLS_KEY_FILE` | - | Certificate and key. Required when `GRPC_TLS_ENABLED=true`. | +| `GRPC_TLS_CERT_FILE`, `GRPC_TLS_KEY_FILE` | None | Certificate and key. Required when `GRPC_TLS_ENABLED=true`. | | `NONCE_TTL_SECONDS` | `600` | How far a gRPC request's timestamp may be from the server's clock; nonces are remembered this long to refuse replays | ## Database -See [Database](../installation/database.md). Keys are 64 hex characters; see [Encryption keys](encryption-keys.md). +See [Database](../installation/database.md) and [Encryption keys](encryption-keys.md). | Setting | Default | Description | | --- | --- | --- | | `DATABASE_DIALECT` | `sqlite` | `sqlite`, `mysql` (MySQL and MariaDB) or `postgres` | | `SQLITE_DATABASE_PATH` | `data/relaysms.db` | SQLite database file | -| `MYSQL_HOST`, `MYSQL_PORT` | `127.0.0.1`, `3306` | MySQL server | -| `MYSQL_USER`, `MYSQL_PASSWORD` | - | Required for `mysql` | +| `MYSQL_HOST` | `127.0.0.1` | MySQL server | +| `MYSQL_PORT` | `3306` | MySQL port | +| `MYSQL_USER`, `MYSQL_PASSWORD` | None | Required for `mysql` | | `MYSQL_DATABASE` | `relaysms` | Required for `mysql`; created if it doesn't exist | -| `POSTGRES_HOST`, `POSTGRES_PORT` | `127.0.0.1`, `5432` | PostgreSQL server | -| `POSTGRES_USER`, `POSTGRES_PASSWORD` | - | Required for `postgres` | +| `POSTGRES_HOST` | `127.0.0.1` | PostgreSQL server | +| `POSTGRES_PORT` | `5432` | PostgreSQL port | +| `POSTGRES_USER`, `POSTGRES_PASSWORD` | None | Required for `postgres` | | `POSTGRES_DATABASE` | `relaysms` | Required for `postgres`; created if it doesn't exist | | `DATABASE_ENCRYPTION_ENABLED` | `true` (`false` if unset) | Encrypts the SQLite file with SQLCipher. No effect on MySQL or PostgreSQL. | -| `DATABASE_ENCRYPTION_KEY` | - | Required for SQLite when encryption is on | +| `DATABASE_ENCRYPTION_KEY` | None | Required for SQLite when encryption is on | | `DATABASE_FIELD_ENCRYPTION_ENABLED` | `true` (`false` if unset) | Encrypts linked accounts' platform tokens | -| `DATABASE_FIELD_ENCRYPTION_KEY` | - | Required when field encryption is on | -| `DATA_ENCRYPTION_KEY` | - | Required. Encrypts the server's private keys, and keys the hash that replaces each sender's address. | +| `DATABASE_FIELD_ENCRYPTION_KEY` | None | Required when field encryption is on | +| `DATA_ENCRYPTION_KEY` | None | Required. Encrypts the server's private keys, and keys the hash that replaces each sender's address. | ## Celery @@ -49,12 +51,20 @@ See [Celery broker](../installation/celery-broker.md). | --- | --- | --- | | `CELERY_BROKER_TYPE` | `sqlite` | `sqlite`, `redis` or `rabbitmq` | | `CELERY_WORKER_CONCURRENCY` | `1` for `sqlite`, else `4` | Tasks the worker runs at once | -| `CELERY_BROKER_DB_PATH`, `CELERY_RESULT_DB_PATH` | `data/celery_broker.db`, `data/celery_results.db` | Broker and result files when `CELERY_BROKER_TYPE=sqlite` | +| `CELERY_BROKER_DB_PATH` | `data/celery_broker.db` | Broker file when `CELERY_BROKER_TYPE=sqlite` | +| `CELERY_RESULT_DB_PATH` | `data/celery_results.db` | Result file when `CELERY_BROKER_TYPE=sqlite` | | `CELERY_REDIS_URL` | `redis://localhost:6379/0` | Broker and result backend when `CELERY_BROKER_TYPE=redis` | | `CELERY_RABBITMQ_URL` | `amqp://guest:guest@localhost:5672//` | Broker when `CELERY_BROKER_TYPE=rabbitmq` | | `CELERY_BEAT_SCHEDULE_PATH` | `data/celerybeat-schedule` | Beat's schedule file | -| `CELERY_CLEANUP_CRON` | `0 */3 * * *` | Cron schedule (5 fields, UTC) for deleting incomplete payloads, ended web sessions, and audit events and adapter jobs past retention, and failing abandoned adapter jobs | -| `CELERY_TOKEN_CLEANUP_CRON` | `0 3 * * *` | Cron schedule (5 fields, UTC) for deleting tokens idle past `TOKEN_IDLE_MAX_AGE_DAYS` and revoking them on their platform | +| `CELERY_CLEANUP_CRON` | `0 */3 * * *` | When to clean up expired data ([below](#cleanup-jobs)) | +| `CELERY_TOKEN_CLEANUP_CRON` | `0 3 * * *` | When to remove idle tokens ([below](#cleanup-jobs)) | + +### Cleanup jobs + +Both schedules are 5-field cron expressions in UTC. + +- `CELERY_CLEANUP_CRON` deletes incomplete payloads, ended web sessions, and audit events and adapter jobs past retention, and fails abandoned adapter jobs. +- `CELERY_TOKEN_CLEANUP_CRON` deletes tokens idle past `TOKEN_IDLE_MAX_AGE_DAYS` and revokes them on their platform. ## Retention @@ -71,7 +81,7 @@ See [Offline publishing](offline-publishing.md). | Setting | Default | Description | | --- | --- | --- | | `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` | `smtp` (empty if unset) | Comma-separated protocols whose offline payloads publish: `https`, `smtp`. Empty allows both. | -| `OFFLINE_PUBLISH_SHARED_SECRET` | - | 64 hex characters. Required when `https` is allowed. | +| `OFFLINE_PUBLISH_SHARED_SECRET` | None | 64 hex characters. Required when `https` is allowed. | ## SMTP transport @@ -80,12 +90,12 @@ See [SMTP transport](smtp-transport.md). | Setting | Default | Description | | --- | --- | --- | | `SMTP_TRANSPORT_ENABLED` | `false` | Runs the email listener | -| `SMTP_IMAP_SERVER`, `SMTP_IMAP_USERNAME`, `SMTP_IMAP_PASSWORD` | - | IMAP host and login. Required when the transport is enabled. | +| `SMTP_IMAP_SERVER`, `SMTP_IMAP_USERNAME`, `SMTP_IMAP_PASSWORD` | None | IMAP host and login. Required when the transport is enabled. | | `SMTP_IMAP_PORT` | `993` | IMAP port | -| `SMTP_IMAP_MAIL_FOLDER` | `INBOX` | Comma-separated folders to watch, e.g. `INBOX,Spam` | -| `SMTP_TLS_CLIENT_CERTIFICATE`, `SMTP_TLS_CLIENT_KEY` | - | mTLS client certificate and key, if the provider requires one. Set both or neither. | -| `SMTP_ALLOWED_SENDERS` | - | Comma-separated addresses and domains allowed to send. Empty allows nothing. | -| `SMTP_TRUSTED_AUTHSERV_ID` | - | `authserv-id` whose `Authentication-Results` verdicts are trusted, e.g. `mx.google.com`. Required when the transport is enabled. | +| `SMTP_IMAP_MAIL_FOLDER` | `INBOX` | Comma-separated folders to watch, such as `INBOX,Spam` | +| `SMTP_TLS_CLIENT_CERTIFICATE`, `SMTP_TLS_CLIENT_KEY` | None | mTLS client certificate and key, if the provider requires one. Set both or neither. | +| `SMTP_ALLOWED_SENDERS` | None | Comma-separated addresses and domains allowed to send. Empty allows nothing. | +| `SMTP_TRUSTED_AUTHSERV_ID` | None | `authserv-id` whose `Authentication-Results` verdicts are trusted, such as `mx.google.com`. Required when the transport is enabled. | | `SMTP_REQUIRE_DKIM`, `SMTP_REQUIRE_SPF` | `true` | Require `dkim=pass` and `spf=pass` | | `SMTP_VERIFY_DKIM_INDEPENDENTLY` | `false` | Also re-verifies the DKIM signature against DNS | @@ -99,7 +109,9 @@ See [Managing adapters](../platforms/manage.md#files). Relative paths are from t | `PLATFORMS_ADAPTERS_VENV_DIR` | `data/platforms/venvs` | Directory for adapters' virtualenvs | | `PLATFORMS_ADAPTERS_CONFIG_DIR` | `data/platforms/config` | Directory for adapters' `credentials.json` files | | `PLATFORMS_ADAPTERS_STATE_DIR` | `data/platforms/state` | Directory for files adapters write, such as databases | -| `PLATFORMS_GITHUB_ORGS` | - | Comma-separated GitHub orgs administrators may install and update adapters from over the REST API. Empty refuses both; `publisher.sh platforms` is unaffected. | +| `PLATFORMS_GITHUB_ORGS` | None | Comma-separated GitHub orgs that administrators may install and update adapters from over the REST API | + +With `PLATFORMS_GITHUB_ORGS` empty, the REST API refuses to install or update adapters; `publisher.sh platforms` is unaffected. ## REST API access @@ -107,12 +119,14 @@ See [REST API](../reference/rest-api.md#authentication). | Setting | Default | Description | | --- | --- | --- | -| `AUTH_WEB_ORIGINS` | - | Comma-separated web client origins allowed to call the API from a browser with a session cookie, e.g. `https://console.example.org`. Must be the same site as the API. Empty when the web client is served from the API's own origin. | +| `AUTH_WEB_ORIGINS` | None | Comma-separated web client origins allowed to use a session cookie, such as `https://console.example.com` | | `AUTH_SESSION_COOKIE_SECURE` | `true` | Sends the session cookie over HTTPS only. Set `false` only for local HTTP development. | | `AUTH_SESSION_IDLE_MINUTES` | `30` | Minutes of inactivity that end a web session | | `AUTH_SESSION_MAX_HOURS` | `12` | Hours after login when a web session ends, even if it's active | | `API_DOCS_ENABLED` | `false` | Serves interactive API docs at `/docs` and the schema at `/openapi.json`. For development. | +`AUTH_WEB_ORIGINS` origins must be on the same site as the API. Leave it empty when the web client is served from the API's own origin. + ## Logging and monitoring See [Observability](../operating/observability.md). @@ -120,9 +134,10 @@ See [Observability](../operating/observability.md). | Setting | Default | Description | | --- | --- | --- | | `LOG_LEVEL` | `info` | `debug`, `info`, `warning`, `error` or `critical` | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | - | OTLP collector URL for traces, metrics and logs, e.g. `http://localhost:4317`. Needs `requirements-observability.txt`. Blank turns all three off. | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | None | OTLP collector URL for traces, metrics and logs, such as `http://localhost:4317`. Needs `requirements-observability.txt`. Blank turns all three off. | | `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | `grpc` or `http/protobuf` | -| `OTEL_TRACES_SAMPLER`, `OTEL_TRACES_SAMPLER_ARG` | `parentbased_traceidratio`, `1.0` | Which traces to keep, and the fraction kept (`0` to `1`) | +| `OTEL_TRACES_SAMPLER` | `parentbased_traceidratio` | Which traces to keep | +| `OTEL_TRACES_SAMPLER_ARG` | `1.0` | Fraction of traces kept, `0` to `1` | | `OTEL_LOGS_EXPORTER` | `otlp` | `otlp` sends logs to the collector; `none` doesn't | | `OTEL_PYTHON_LOG_CORRELATION` | `true` | Adds trace and span IDs to log records | -| `UPTIME_KUMA_WORKER_PUSH_URL`, `UPTIME_KUMA_SMTP_PUSH_URL` | - | Uptime Kuma push URLs for the worker and email listener heartbeats. Blank turns them off. | +| `UPTIME_KUMA_WORKER_PUSH_URL`, `UPTIME_KUMA_SMTP_PUSH_URL` | None | Uptime Kuma push URLs for the worker and email listener heartbeats. Blank turns them off. | diff --git a/docs/configuration/smtp-transport.md b/docs/configuration/smtp-transport.md index 6bd8b76..9a95c52 100644 --- a/docs/configuration/smtp-transport.md +++ b/docs/configuration/smtp-transport.md @@ -1,50 +1,67 @@ # SMTP transport -Publishes RelaySMS payloads that gateway clients send by email. The listener watches a mailbox over IMAP, checks each sender, and queues the payload for publication. +Use this when gateway clients relay payloads by email instead of HTTPS. The listener watches a mailbox over IMAP, checks each sender, and queues the payload for publication. -## Message format +## Set it up -The body of each email must be a JSON object: +1. Create a mailbox that the gateway clients email, with IMAP access. +2. In `.env`, set `SMTP_IMAP_SERVER`, `SMTP_IMAP_USERNAME` and `SMTP_IMAP_PASSWORD`. To also watch a spam folder, set `SMTP_IMAP_MAIL_FOLDER`, such as `INBOX,Spam`. +3. Set `SMTP_ALLOWED_SENDERS` to the addresses the gateway clients send from ([allowlist](#allowlist)). +4. Set `SMTP_TRUSTED_AUTHSERV_ID` to the `authserv-id` the mailbox's server writes: the first word of the `Authentication-Results` header in a received email, such as `mx.google.com` ([authentication](#authentication)). +5. Set `SMTP_TRANSPORT_ENABLED=true`, then check and restart: -```json -{ - "address": "5447c1f50558292bd9df723f9fdc0b06b892199c7dcfaad5164f2d94dfd3470a", - "dialing_code": "237", - "text": "" -} + ```bash + sudo ./manage.sh check + sudo ./manage.sh restart + ``` + +[Settings reference](settings.md#smtp-transport) lists every `SMTP_*` setting. + +## Check it + +```bash +sudo ./manage.sh logs --unit smtp --no-follow --lines 20 ``` -| Field | Type | Description | -| :--- | :--- | :--- | -| address | string | Sender ID from the gateway client, or an E.164 phone number | -| dialing_code | string | Optional. Sender's country calling code, for publication stats | -| text | string | Base64-encoded serialized payload | +You should see `Connected to mailbox `. ## How it works -1. Waits for unseen mail in `SMTP_IMAP_MAIL_FOLDER` with IMAP IDLE. List several folders, such as `INBOX,Spam`, to catch mail a provider misfiled. -2. Rejects each email unless the sender is allowlisted and authenticated (see [Security](#security)). +1. Waits for unseen mail in `SMTP_IMAP_MAIL_FOLDER` with IMAP IDLE. +2. Rejects each email unless the sender is allowlisted and authenticated. 3. Parses the body as JSON and validates the payload. -4. Queues the payload for publication. +4. Queues the payload for publication, tagged with protocol `smtp`. Whether offline payloads from email publish depends on [offline publishing](offline-publishing.md). 5. Deletes each handled email. An email that hits an unexpected error stays in the folder and is retried on the next cycle. -## Security +The `relaysms-publisher-smtp` service exits at once while `SMTP_TRANSPORT_ENABLED` is off. -Two checks must both pass before a message is queued: +### Allowlist -**Allowlist:** `SMTP_ALLOWED_SENDERS` lists exact addresses and domains, comma-separated. Empty allows nothing. +`SMTP_ALLOWED_SENDERS` lists exact addresses and domains, comma-separated. Empty allows nothing. > [!WARNING] > A domain entry trusts every authenticated sender on that domain, since DKIM vouches for the domain, not the mailbox. List an exact address when only one sender should be trusted. -**Authentication:** the listener can't re-check SPF, which needs the sender's IP. It reads the `Authentication-Results` header ([RFC 8601](https://www.rfc-editor.org/rfc/rfc8601)) that the mailbox's own server added, and only from the `authserv-id` in `SMTP_TRUSTED_AUTHSERV_ID`, since a sender can forge such a header from anyone else. `SMTP_REQUIRE_DKIM` and `SMTP_REQUIRE_SPF` choose the verdicts required. `SMTP_VERIFY_DKIM_INDEPENDENTLY=true` also re-verifies the DKIM signature against DNS. +### Authentication -## Configuration +The listener can't re-check SPF, which needs the sender's IP. It reads the `Authentication-Results` header ([RFC 8601](https://www.rfc-editor.org/rfc/rfc8601)) that the mailbox's own server added, and only from the `authserv-id` in `SMTP_TRUSTED_AUTHSERV_ID`, since a sender can forge such a header from anyone else. `SMTP_REQUIRE_DKIM` and `SMTP_REQUIRE_SPF` choose the verdicts required. `SMTP_VERIFY_DKIM_INDEPENDENTLY=true` also re-verifies the DKIM signature against DNS. -Settings: [SMTP transport](settings.md#smtp-transport). +## Message format + +The body of each email must be a JSON object: -The listener tags what it queues with protocol `smtp`. Whether offline payloads from email publish depends on [offline publishing](offline-publishing.md). +```json +{ + "address": "5447c1f50558292bd9df723f9fdc0b06b892199c7dcfaad5164f2d94dfd3470a", + "dialing_code": "237", + "text": "" +} +``` -## Running +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `address` | string | Yes | Sender ID from the gateway client, or an E.164 phone number | +| `dialing_code` | string | No | Sender's country calling code, for publication stats | +| `text` | string | Yes | Base64-encoded serialized payload | -Set `SMTP_TRANSPORT_ENABLED=true`, then run `sudo ./manage.sh restart`. The `relaysms-publisher-smtp` service runs the listener and exits at once while the setting is off. +Next: [Offline publishing](offline-publishing.md). diff --git a/docs/developers/reference-client.md b/docs/developers/reference-client.md index 483899c..f2f66a4 100644 --- a/docs/developers/reference-client.md +++ b/docs/developers/reference-client.md @@ -14,9 +14,9 @@ python -m tools.client [ARGS] | `--host`, `--port` | `127.0.0.1`, `6000` | gRPC server | | `--tls` | off | Use TLS for gRPC | | `--rest-api` | `http://localhost:16000` | REST API base URL | -| `--platform`, `-p` | | Platform name, such as `gmail` or `telegram` | -| `--phone-number` | | Phone number, for phone number-based authentication (PNBA, e.g. Telegram) | -| `--request-identifier` | | Optional request identifier | +| `--platform`, `-p` | None | Platform name, such as `gmail` or `telegram` | +| `--phone-number` | None | Phone number, for phone number-based authentication (PNBA), such as for Telegram | +| `--request-identifier` | None | Optional request identifier | ## Linking accounts @@ -37,8 +37,7 @@ python -m tools.client get-pnba-code --platform telegram --phone-number +2371234 python -m tools.client exchange-pnba-code --platform telegram --phone-number +237123456789 --code ``` -> [!NOTE] -> With several stored tokens, `sync-keys` and the revoke commands ask which one to use. Pass `--token` to choose up front. +With several stored tokens, `sync-keys` and the revoke commands ask which one to use. Pass `--token` to choose up front. ## Publishing @@ -69,8 +68,6 @@ python -m tools.client send --offline --platform rmail --address +237123456789 \ | `--offline` | Uses offline-first encryption, which needs no linked token (`rmail` only) | | `--tag` | The server's `OFFLINE_PUBLISH_SHARED_SECRET`, when it sets one | -> [!NOTE] -> Each online send uses up one ephemeral key pair. Run `sync-keys` when the pool runs low. Offline sends use none. +Each online send uses up one ephemeral key pair. Run `sync-keys` when the pool runs low. Offline sends use none. -> [!TIP] -> `--shuffle --dry-run` shows the segment order before a live send, to check the server reassembles out-of-order segments. +`--shuffle --dry-run` shows the segment order before a live send, to check the server reassembles out-of-order segments. diff --git a/docs/gateway-clients/connect-a-relay.md b/docs/gateway-clients/connect-a-relay.md new file mode 100644 index 0000000..5590e16 --- /dev/null +++ b/docs/gateway-clients/connect-a-relay.md @@ -0,0 +1,36 @@ +# Connecting a relay + +A gateway client receives the SMS that people send and relays each payload to the Publisher's `POST /v1/publications`. There are two: + +| Gateway client | Runs on | Use it for | +| --- | --- | --- | +| [Deku SMS](https://f-droid.org/packages/com.afkanerd.deku/) | An Android phone with a SIM card | A local number, with no SMS provider | +| [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) | A server, behind a Twilio number | A number that doesn't depend on a phone staying online | + +## Requirements + +- The Publisher's REST API reachable over HTTPS, behind the [reverse proxy](../installation/reverse-proxy.md) +- The number [registered as a gateway client](manage.md), so the RelaySMS app lists it + +## Deku SMS on Android + +1. Install [Deku SMS](https://f-droid.org/packages/com.afkanerd.deku/) on the phone and make it the default SMS app. +2. In its gateway settings, forward incoming SMS to `https:///v1/publications`. +3. Keep the phone powered, with mobile data or Wi-Fi. + +## Twilio + +1. Install [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) as its README describes. +2. In its `.env`, set `FORWARD_URLS_JSON=https:///v1/publications`. +3. If the Publisher allows [offline publishing](../configuration/offline-publishing.md) over `https`, set `FORWARD_TAG` to its `OFFLINE_PUBLISH_SHARED_SECRET`. +4. In the Twilio console, under the number's Messaging > A message comes in, set `https:///twilio-sms` with `HTTP POST`. + +## Check it + +Send an SMS from the RelaySMS app to the number, then follow the worker's log on the Publisher: + +```bash +sudo ./manage.sh logs --unit worker +``` + +You should see `Published via ''.` To test without a phone, the [reference client](../developers/reference-client.md#publishing) posts a payload the way a gateway client does. diff --git a/docs/gateway-clients/manage.md b/docs/gateway-clients/manage.md index 1ba7032..5934f95 100644 --- a/docs/gateway-clients/manage.md +++ b/docs/gateway-clients/manage.md @@ -1,20 +1,26 @@ # Managing gateway clients -Gateway clients are the phone numbers that relay SMS to this server. The enabled ones are listed publicly at `GET /v1/gateway-clients`. Credentials with `gc:read` and `gc:write` can manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients). Every change is recorded in the [audit log](../reference/rest-api.md#audit-events). +A gateway client is a phone number that relays the SMS people send to this server: an Android phone running Deku SMS, or a Twilio number. Register each number here so the RelaySMS app lists it, then [connect its relay](connect-a-relay.md). The enabled ones are listed publicly at `GET /v1/gateway-clients`. Credentials with `gc:read` and `gc:write` can manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients). Every change is recorded in the [audit log](../reference/rest-api.md#audit-events). ## Commands +| Command | Does | +| --- | --- | +| `gateway-clients suggest ` | Shows the number's details and PLMN candidates; saves nothing | +| `gateway-clients create --msisdn --protocols ` | Registers a number; also takes `--country`, `--operator` and `--operator-code` | +| `gateway-clients list` | Lists them; filter with `--msisdn`, `--country` or `--operator` | +| `gateway-clients update ` | Changes `--country`, `--operator`, `--operator-code` or `--protocols` | +| `gateway-clients disable ` | Hides it from the public list | +| `gateway-clients enable ` | Lists it publicly again | +| `gateway-clients delete ` | Removes it | +| `gateway-clients countries` | Lists the countries that have a gateway client | +| `gateway-clients operators --country ` | Lists the operators of that country's gateway clients | +| `gateway-clients mcc-mnc list` | Searches the PLMN table; filter with `--country-code`, `--network` or `--iso` | + +Prefix each with `sudo ./publisher.sh`. `` is comma-separated, such as `https,smtp`. For example: + ```bash -sudo ./publisher.sh gateway-clients suggest # details and PLMN candidates; saves nothing -sudo ./publisher.sh gateway-clients create --msisdn --protocols [--country ] [--operator ] [--operator-code ] -sudo ./publisher.sh gateway-clients list [--msisdn ] [--country ] [--operator ] -sudo ./publisher.sh gateway-clients update [--country ] [--operator ] [--operator-code ] [--protocols ] -sudo ./publisher.sh gateway-clients disable # hide it from the public list -sudo ./publisher.sh gateway-clients enable -sudo ./publisher.sh gateway-clients delete -sudo ./publisher.sh gateway-clients countries -sudo ./publisher.sh gateway-clients operators --country -sudo ./publisher.sh gateway-clients mcc-mnc list [--country-code ] [--network ] [--iso ] # search the PLMN table +sudo ./publisher.sh gateway-clients create --msisdn +237600000000 --protocols https ``` `create` fills in any of `--country`, `--operator` and `--operator-code` you leave out with what `suggest` finds. It fills the PLMN code (MCC+MNC) only when every candidate shares it; otherwise run `suggest`, pick a candidate and pass it with `--operator-code`. @@ -31,8 +37,7 @@ Names are compared word by word, ignoring case, accents, company-type words ("Te | `region` | Every operator in the country, because the carrier is unknown (common for US and Canadian numbers) or matched nothing | | `none` | None: the number couldn't be placed. Enter the details yourself. | -> [!TIP] -> A PLMN missing from the snapshot can still be entered with `--operator-code`. +A PLMN missing from the snapshot can still be entered with `--operator-code`. ## Upgrading from the JSON registry diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md index cf012c6..dd8bd04 100644 --- a/docs/getting-started/quick-start.md +++ b/docs/getting-started/quick-start.md @@ -1,6 +1,26 @@ # Quick start -Runs the Publisher on your machine with Docker, to try it before installing it on a server. You need Git, Docker with the Compose plugin, `make` and `openssl`. +Runs the Publisher on your machine with Docker, to try it before installing it on a server. + +## Requirements + +- **Docker with the Compose plugin**: [Docker Engine](https://docs.docker.com/engine/install/) on Linux, or [Docker Desktop](https://docs.docker.com/desktop/), which includes Compose, on macOS and Windows. On Linux, [add yourself to the `docker` group](https://docs.docker.com/engine/install/linux-postinstall/) or prefix the `docker` commands with `sudo`. +- **Git, `make` and `openssl`**: on macOS, `xcode-select --install` provides Git and `make`, and `openssl` ships with the system. On Windows, run the steps in [WSL](https://learn.microsoft.com/en-us/windows/wsl/install). On Ubuntu or Debian: + + ```bash + sudo apt-get install -y git make openssl + ``` + +Check them: + +```bash +docker compose version +git --version +make --version +openssl version +``` + +## Steps 1. Clone the repository with its submodule: @@ -32,11 +52,12 @@ Runs the Publisher on your machine with Docker, to try it before installing it o 5. Create an administrator credential: ```bash - docker compose exec publisher python -m publisher creds create --username admin --administrator + docker compose exec publisher \ + python -m publisher creds create --username admin --administrator ``` > [!IMPORTANT] - > The password is printed once. Store it before closing the terminal. + > The password is printed only once. 6. Use it on the REST API: @@ -50,8 +71,8 @@ To stop the Publisher, run `docker compose down`. To start again from scratch, a ## Next steps +The pages below run the admin CLI as `sudo ./publisher.sh`; with Docker, use `docker compose exec publisher python -m publisher` instead. + - [Add a platform](../platforms/manage.md) - [Register a gateway client](../gateway-clients/manage.md) - [Install on a server](../installation/server.md) for production - -Those pages run the admin CLI as `sudo ./publisher.sh`; with Docker, use `docker compose exec publisher python -m publisher` instead. diff --git a/docs/installation/celery-broker.md b/docs/installation/celery-broker.md index 4b68f61..0472836 100644 --- a/docs/installation/celery-broker.md +++ b/docs/installation/celery-broker.md @@ -14,23 +14,34 @@ The REST API and the email listener queue publications for the Celery worker thr A setup script installs RabbitMQ, creates a vhost and user, and writes the broker settings to `.env`: +During install: + ```bash -sudo ./install.sh --setup-broker rabbitmq # during install -sudo ./scripts/setup-rabbitmq.sh --broker-vhost relaysms --broker-user relaysms # afterwards +sudo ./install.sh --setup-broker rabbitmq ``` -`--broker-password PASS` sets the password instead of generating one. To use a broker you already have, add `--existing` (`--broker-existing` on `install.sh`) with `--broker-host`. The script checks the credentials against the management API (port `15672`, or `--broker-mgmt-port`) and changes nothing on the broker. +On an existing install, from the install directory: + +```bash +sudo ./scripts/setup-rabbitmq.sh --broker-vhost relaysms --broker-user relaysms +``` + +`--broker-password ` sets the password instead of generating one. To use a broker you already have, add `--existing` (`--broker-existing` on `install.sh`) with `--broker-host`. The script checks the credentials against the management API (port `15672`, or `--broker-mgmt-port`) and changes nothing on the broker. ## By hand +For RabbitMQ, in `.env`: + ```bash CELERY_BROKER_TYPE=rabbitmq -CELERY_RABBITMQ_URL=amqp://relaysms:your-password@localhost:5672/relaysms +CELERY_RABBITMQ_URL=amqp://relaysms:@localhost:5672/relaysms ``` +For Redis: + ```bash CELERY_BROKER_TYPE=redis CELERY_REDIS_URL=redis://localhost:6379/0 ``` -After either, restart the services with `sudo ./manage.sh restart`. +Then restart the services with `sudo ./manage.sh restart`. diff --git a/docs/installation/database.md b/docs/installation/database.md index dd7dde8..7eb25be 100644 --- a/docs/installation/database.md +++ b/docs/installation/database.md @@ -10,38 +10,61 @@ Sensitive fields are also encrypted in every dialect; see [Encryption keys](../c ## MySQL, MariaDB or PostgreSQL -A setup script installs the server, creates the database and user, and writes the connection details to `.env`. Running it again without `--db-password` gives the user a new password; restart the services afterwards. The MySQL script installs `mysql-server`, which only Ubuntu ships; on Debian, set up MariaDB yourself and pass `--db-existing`. +A setup script installs the server, creates the database and user, and writes the connection details to `.env`. Running it again without `--db-password` gives the user a new password. The MySQL script installs `mysql-server`, which only Ubuntu ships; on Debian, set up MariaDB yourself and pass `--db-existing`. + +During install: ```bash -sudo ./install.sh --setup-db postgres # during install -sudo ./scripts/setup-postgres.sh --db-name relaysms --db-user relaysms # afterwards -sudo ./scripts/setup-mysql.sh --db-name relaysms --db-user relaysms +sudo ./install.sh --setup-db postgres ``` -`--db-password PASS` sets the password instead of generating one. +On an existing install, from the install directory: + +```bash +sudo ./scripts/setup-postgres.sh --db-name relaysms --db-user relaysms +``` + +For MySQL or MariaDB, use `--setup-db mysql` or `scripts/setup-mysql.sh` the same way. + +`--db-password ` sets the password instead of generating one. To use a server you already have, pass its details. The script only checks the connection and writes `.env`; it never changes the server: ```bash sudo ./install.sh --setup-db postgres --db-existing \ --db-host db.example.com --db-port 5432 \ - --db-name relaysms --db-user relaysms --db-password 'your-password' + --db-name relaysms --db-user relaysms --db-password '' +``` +On an existing install: + +```bash sudo ./scripts/setup-postgres.sh --existing \ - --db-host db.example.com --db-name relaysms --db-user relaysms --db-password 'your-password' + --db-host db.example.com --db-name relaysms --db-user relaysms --db-password '' ``` -Or set it by hand: +Or set it by hand in `.env`. For PostgreSQL: ```bash -DATABASE_DIALECT=postgres # or mysql, for MySQL and MariaDB -POSTGRES_HOST=127.0.0.1 # MYSQL_HOST, MYSQL_PORT, ... for mysql +DATABASE_DIALECT=postgres +POSTGRES_HOST=127.0.0.1 POSTGRES_PORT=5432 POSTGRES_USER=relaysms -POSTGRES_PASSWORD=your-password +POSTGRES_PASSWORD= POSTGRES_DATABASE=relaysms ``` +For MySQL or MariaDB: + +```bash +DATABASE_DIALECT=mysql +MYSQL_HOST=127.0.0.1 +MYSQL_PORT=3306 +MYSQL_USER=relaysms +MYSQL_PASSWORD= +MYSQL_DATABASE=relaysms +``` + Then apply the migrations and restart: ```bash @@ -49,5 +72,4 @@ sudo ./manage.sh migrate sudo ./manage.sh restart ``` -> [!WARNING] -> The new database starts empty. Switching doesn't copy data from SQLite. +The new database starts empty: switching doesn't copy the data from SQLite. diff --git a/docs/installation/docker.md b/docs/installation/docker.md index 7470066..cf2d12c 100644 --- a/docs/installation/docker.md +++ b/docs/installation/docker.md @@ -1,6 +1,8 @@ # Install with Docker -`docker-compose.yml` runs every service in one container, with its data in `./data` and its settings in `.env` (see the [settings reference](../configuration/settings.md)): +`docker-compose.yml` runs every service in one container, with its data in `./data` and its settings in `.env` (see the [settings reference](../configuration/settings.md)). It needs the tools in the [quick start's requirements](../getting-started/quick-start.md#requirements). + +To install: ```bash git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git @@ -21,3 +23,5 @@ Back up `.env` with `./data`: its [encryption keys](../configuration/encryption- git pull --recurse-submodules docker compose up -d --build ``` + +Next: [Settings reference](../configuration/settings.md). diff --git a/docs/installation/requirements.md b/docs/installation/requirements.md index a036925..af3f4ef 100644 --- a/docs/installation/requirements.md +++ b/docs/installation/requirements.md @@ -1,12 +1,10 @@ # Requirements -| Need | Details | -| --- | --- | -| Operating system | Ubuntu 24.04 or Debian 13, or any Linux host with Docker | -| Python | 3.12 or newer. The installer installs it, with the other system packages and Rust. | -| Database | SQLite (the default), MySQL 8.0.28 or newer, MariaDB, or PostgreSQL 12 or newer. See [Database](database.md). | -| Domain | A DNS name pointing at the server, for the TLS certificate. See [Reverse proxy and TLS](reverse-proxy.md). | -| Inbound ports | 80 and 443 for nginx. On a server install, the services listen on `127.0.0.1` only. | -| Outbound access | `github.com`, `pypi.org`, the Rust installer and the system's package mirrors to install the Publisher and its adapters; Let's Encrypt for certificates; each platform's API to publish | -| Platform credentials | From each platform you offer, e.g. a Google Cloud OAuth client for Gmail. See [Managing adapters](../platforms/manage.md). | -| Gateway client | At least one phone number that relays SMS to the REST API or a mailbox. See [Managing gateway clients](../gateway-clients/manage.md). | +- **Operating system**: Ubuntu 24.04 or Debian 13, or any Linux host with Docker. +- **Python**: 3.12 or newer. The installer installs it, with the other system packages and Rust. +- **Database**: SQLite (the default), MySQL 8.0.28 or newer, MariaDB, or PostgreSQL 12 or newer. See [Database](database.md). +- **Domain**: a DNS name pointing at the server, for the TLS certificate. See [Reverse proxy and TLS](reverse-proxy.md). +- **Inbound ports (firewall)**: 80 and 443 for nginx. On a server install, the services listen on `127.0.0.1` only. Docker publishes `PORT` (16000) and `GRPC_PORT` (6000). +- **Outbound access**: `github.com`, `pypi.org`, the Rust installer and the system's package mirrors to install the Publisher and its adapters; Let's Encrypt for certificates; each platform's API to publish. +- **Platform credentials**: from each platform you offer, such as a Google Cloud OAuth client for Gmail. See [Provider guides](../platforms/manage.md#provider-guides). +- **Gateway client**: at least one phone number that relays SMS, on an Android phone running Deku SMS or a Twilio number. See [Connecting a relay](../gateway-clients/connect-a-relay.md). diff --git a/docs/installation/reverse-proxy.md b/docs/installation/reverse-proxy.md index 2297561..285d870 100644 --- a/docs/installation/reverse-proxy.md +++ b/docs/installation/reverse-proxy.md @@ -1,13 +1,6 @@ # Reverse proxy and TLS -The services listen on `127.0.0.1`. nginx serves them on port 443 with a Let's Encrypt certificate, from [the site template](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/deploy/nginx/relaysms-publisher-nginx.conf.template): - -| Path | Goes to | -| --- | --- | -| `/publisher.v3.Publisher`, `/grpc.health.v1.Health` | gRPC, on `GRPC_PORT` | -| Everything else | REST, on `PORT` | - -The site also rate-limits logins and requests carrying HTTP Basic credentials. +The services listen on `127.0.0.1`. nginx serves them on port 443 with a Let's Encrypt certificate, from [the site template](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/deploy/nginx/relaysms-publisher-nginx.conf.template). It sends `/publisher.v3.Publisher` and `/grpc.health.v1.Health` to gRPC on `GRPC_PORT`, and everything else to REST on `PORT`, and rate-limits logins and requests carrying HTTP Basic credentials. ## With the installer diff --git a/docs/installation/server.md b/docs/installation/server.md index 544880f..9c83ad4 100644 --- a/docs/installation/server.md +++ b/docs/installation/server.md @@ -5,14 +5,23 @@ The installer sets up the Publisher on Ubuntu 24.04 or Debian 13 under systemd, ## Run the installer ```bash -curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | sudo bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash +``` + +To read the script before running it: + +```bash +curl -fsSLO https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh +less install.sh +sudo bash install.sh ``` It asks about each option you don't pass as a flag. To choose up front, pass flags after `bash -s --`: ```bash -curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ - sudo bash -s -- --site-name publisher.example.com --setup-db postgres +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash -s -- --site-name publisher.example.com --setup-db postgres ``` | Flag | Does | @@ -40,6 +49,21 @@ The installer: Running it again updates the install in place and keeps its `.env`. Run it as the same user as the first time; it refuses another service user. +## Serving the manual + +`--docs-site-name` builds this manual into `book/` and serves it with nginx on its own domain and certificate. `manage.sh update` rebuilds it. + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash -s -- --site-name publisher.example.com --docs-site-name docs.publisher.example.com +``` + +On an install that's already running, add it from the install directory: + +```bash +sudo ./manage.sh docs docs.publisher.example.com +``` + ## After installing 1. Go to the install directory and check that the services are up: @@ -47,10 +71,10 @@ Running it again updates the install in place and keeps its `.env`. Run it as th ```bash cd /opt/relaysms/relaysms-publisher sudo ./manage.sh status - curl http://127.0.0.1:16000/health # or https:///health with nginx + curl http://127.0.0.1:16000/health ``` - You should see `{"status":"ok"}`. + You should see `{"status":"ok"}`, also at `https:///health` once nginx is set up. 2. Set what you need in `.env` (see the [settings reference](../configuration/settings.md)), then check it and restart: @@ -66,19 +90,10 @@ Running it again updates the install in place and keeps its `.env`. Run it as th ``` > [!IMPORTANT] - > The password is printed once. Store it before closing the terminal. + > The password is printed only once. 4. Back up `.env`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. 5. [Add the platform adapters](../platforms/manage.md) you want to offer, and [register the gateway clients](../gateway-clients/manage.md) that relay SMS to this server. -[Services](../operating/services.md) covers `manage.sh` and `publisher.sh`. - -## Serving the manual - -`--docs-site-name` builds this manual into `book/` and serves it with nginx on its own domain and certificate. `manage.sh update` rebuilds it. - -```bash -curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ - sudo bash -s -- --site-name publisher.example.com --docs-site-name docs.publisher.example.com -``` +Next: [Services](../operating/services.md). diff --git a/docs/operating/observability.md b/docs/operating/observability.md index ac7ba54..ce707b4 100644 --- a/docs/operating/observability.md +++ b/docs/operating/observability.md @@ -2,126 +2,118 @@ Self-hosted tracing, metrics and log export (SigNoz) and uptime monitoring (Uptime Kuma) for the Publisher's five services. Both are optional; tracing stays off while `OTEL_EXPORTER_OTLP_ENDPOINT` is blank. -## Setup +## Set up + +During install: + +```bash +sudo ./install.sh --setup-observability +``` + +On an existing install, from the install directory: ```bash -sudo ./install.sh --setup-observability # during install -sudo ./scripts/setup-observability.sh # on an existing install +sudo ./scripts/setup-observability.sh ``` | `install.sh` | `setup-observability.sh` | Does | | --- | --- | --- | -| `--observability-site-name DOMAIN` | `--site-name DOMAIN` | Puts nginx and a Let's Encrypt certificate in front of SigNoz | -| `--observability-kuma-site-name DOMAIN` | `--kuma-site-name DOMAIN` | The same for Uptime Kuma, on a different domain | -| - | `--signoz-email`, `--signoz-password` | SigNoz admin account | -| - | `--signoz-smtp-host`, `-port`, `-username`, `-password`, `-from` | SMTP relay for SigNoz's email alerts | -| - | `--skip-tracing` | Leaves tracing off | +| `--observability-site-name ` | `--site-name ` | Puts nginx and a Let's Encrypt certificate in front of SigNoz | +| `--observability-kuma-site-name ` | `--kuma-site-name ` | The same for Uptime Kuma, on a different domain | +| None | `--signoz-email`, `--signoz-password` | SigNoz admin account | +| None | `--signoz-smtp-host`, `--signoz-smtp-port`, `--signoz-smtp-username`, `--signoz-smtp-password`, `--signoz-smtp-from` | SMTP relay for SigNoz's email alerts | +| None | `--skip-tracing` | Leaves tracing off | -`scripts/setup-observability.sh --help` lists every flag. The script installs Docker if it's missing. +`scripts/setup-observability.sh --help` lists every flag. The script: -## SigNoz +- installs Docker if it's missing; +- writes `observability/signoz/casting.yaml` from its template, binds the SigNoz UI and OTLP ports to `127.0.0.1`, starts SigNoz and creates the admin account; +- starts Uptime Kuma on `127.0.0.1:3001`, or the port you choose; +- installs `requirements-observability.txt` and sets `OTEL_EXPORTER_OTLP_ENDPOINT`, unless given `--skip-tracing`; +- with `--site-name` or `--kuma-site-name`, writes and enables the nginx sites and runs Certbot. + +Set retention in the SigNoz UI, under Settings > Workspace > Retention Controls. + +> [!IMPORTANT] +> `casting.yaml` is generated only once. To change a running install, edit `observability/signoz/casting.yaml` and run `foundryctl cast -f observability/signoz/casting.yaml` again. It recreates only the `signoz` container; Postgres and ClickHouse keep their data. -**Automated**: writes `observability/signoz/casting.yaml` from its template (Postgres credentials, ports, SMTP), binds the UI and OTLP ports to `127.0.0.1`, points the collector's opamp at SigNoz so the OTLP receiver opens, starts SigNoz and creates the admin account. +## Uptime Kuma monitors -**Manual** (needs Docker Engine with the Compose plugin): +Open Uptime Kuma through its domain, or through an SSH tunnel at : ```bash -curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_INSTALL_DIR=/usr/local/bin bash +ssh -L 3001:127.0.0.1:3001 @ +``` -sed \ - -e "s/__POSTGRES_DB__/signoz/" -e "s/__POSTGRES_USER__/signoz/" \ - -e "s/__POSTGRES_PASSWORD__/$(openssl rand -hex 24)/" \ - -e "s/__SIGNOZ_PORT__/8090/" -e "s/__OTLP_GRPC_PORT__/4317/" -e "s/__OTLP_HTTP_PORT__/4318/" \ - -e "s/__SMTP_SMARTHOST__//" -e "s/__SMTP_USERNAME__//" -e "s/__SMTP_PASSWORD__//" -e "s/__SMTP_FROM__//" \ - -e "s|__SIGNOZ_EXTERNAL_URL__|http://localhost:8090|" \ - observability/signoz/casting.yaml.template > observability/signoz/casting.yaml +Then add a monitor for each service. -foundryctl cast -f observability/signoz/casting.yaml -docker compose \ - -f pours/deployment/compose.yaml \ - -f observability/signoz/docker-compose.override.yml \ - up -d +### REST -until curl -fs http://127.0.0.1:8090/api/v1/version | grep -q setupCompleted; do sleep 2; done -curl -X POST http://127.0.0.1:8090/api/v1/register \ - -H "Content-Type: application/json" \ - -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@relaysms.local","password":"<12+ chars, upper, lower, number, symbol>"}' -``` +Monitor Type "HTTP(s)", URL `https:///health`. -The template stores SigNoz's metadata in Postgres (`metastore.kind: postgres`); set `sqlite` for a single node. Set retention in the UI, under Settings > Workspace > Retention Controls. +### gRPC -> [!IMPORTANT] -> `casting.yaml` is generated only once. To change a running install, edit `observability/signoz/casting.yaml` and run `foundryctl cast -f observability/signoz/casting.yaml` again. It recreates only the `signoz` container; Postgres and ClickHouse keep their data. +1. Choose Monitor Type "GRPC(S) - Keyword". +2. Set gRPC URL to `:443` and turn on Enable TLS. +3. Set Service Name to `grpc.health.v1.Health` and Method to `check`. +4. Set gRPC Body to `{"service":"publisher.v3.Publisher"}` and Keyword to `SERVING`. +5. Paste this into Protobuf: -## Uptime Kuma +
+Health check protobuf -**Automated**: starts the container on `127.0.0.1:3001`, or the port you choose, which it saves to `observability/uptime-kuma/.env`. +```proto +syntax = "proto3"; +package grpc.health.v1; -**Manual**: `docker compose -f observability/uptime-kuma/docker-compose.yml up -d` +message HealthCheckRequest { + string service = 1; +} -Open it through its own domain ([below](#reverse-proxy)) or an SSH tunnel (`ssh -L 3001:127.0.0.1:3001 @`, then ). Create the admin account on the first visit, then add these monitors: +message HealthCheckResponse { + enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; + SERVICE_UNKNOWN = 3; + } + ServingStatus status = 1; +} -- **REST**: Monitor Type "HTTP(s)", URL `https:///health`. -- **gRPC**: Monitor Type "GRPC(S) - Keyword", with: - - gRPC URL: `:443` - - Enable TLS: on - - Service Name: `grpc.health.v1.Health` - - Method: `check` - - gRPC Body: `{"service":"publisher.v3.Publisher"}` - - Keyword: `SERVING` - - Protobuf: +service Health { + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); +} +``` - ```proto - syntax = "proto3"; - package grpc.health.v1; +
- message HealthCheckRequest { - string service = 1; - } +### Worker - message HealthCheckResponse { - enum ServingStatus { - UNKNOWN = 0; - SERVING = 1; - NOT_SERVING = 2; - SERVICE_UNKNOWN = 3; - } - ServingStatus status = 1; - } +Monitor Type "Push", interval just over 60s. Set `UPTIME_KUMA_WORKER_PUSH_URL` in `.env` to the push URL Kuma shows, then restart the services. Beat schedules this heartbeat, so the monitor also goes down when Beat stops. - service Health { - rpc Check(HealthCheckRequest) returns (HealthCheckResponse); - } - ``` +### SMTP listener -- **Worker**: Monitor Type "Push", interval just over 60s. Set `UPTIME_KUMA_WORKER_PUSH_URL` in `.env` to the push URL Kuma shows, then run `sudo ./manage.sh restart`. Beat schedules this heartbeat, so the monitor also goes down when Beat stops. -- **SMTP listener**: as for the worker, with an interval just over 20s and `UPTIME_KUMA_SMTP_PUSH_URL`. +As for the worker, with an interval just over 20s and `UPTIME_KUMA_SMTP_PUSH_URL`. -## Reverse proxy +## Alerts -**Automated**: `--site-name` and `--kuma-site-name` on `setup-observability.sh` write and enable the nginx sites and run Certbot. +In the SigNoz UI, go to Alerts > New Alert > Logs, paste a filter below into the search bar, set the threshold to count > 0 over a rolling window, and attach a notification channel. Keep the parentheses: `AND` binds tighter than `OR`. -**Manual**: +Publish failures, adapter errors included: -```bash -sed -e "s/__SERVER_NAME__//g" \ - observability/nginx-observability.conf.template \ - > /etc/nginx/sites-available/.conf -ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ - -sed -e "s/__SERVER_NAME__//g" \ - observability/nginx-uptime-kuma.conf.template \ - > /etc/nginx/sites-available/.conf -ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ - -nginx -t && systemctl reload nginx -certbot --nginx -d -certbot --nginx -d +``` +service.name = 'relaysms-publisher-worker' AND (body CONTAINS 'Failed to process payload' OR body CONTAINS 'Failed to publish message' OR body CONTAINS 'An unexpected error occurred during task processing') +``` + +Key management failures: + +``` +body CONTAINS 'Failed to generate server identity keys' OR body CONTAINS 'Token hash missing' ``` ## Turning tracing on and off -`setup-observability.sh` installs `requirements-observability.txt` and sets the endpoint unless given `--skip-tracing`. To do it by hand, from the install directory: +To turn tracing on by hand, from the install directory: ```bash sudo venv/bin/pip install --quiet -r requirements-observability.txt @@ -129,20 +121,61 @@ sudo sed -i 's|^OTEL_EXPORTER_OTLP_ENDPOINT=.*|OTEL_EXPORTER_OTLP_ENDPOINT=http: sudo ./manage.sh restart ``` -Blank the value and restart to turn tracing off. +To turn it off, blank `OTEL_EXPORTER_OTLP_ENDPOINT` and restart. -## Alerts +## Setting up by hand -In the SigNoz UI, go to Alerts > New Alert > Logs, paste a filter below into the search bar, set the threshold to count > 0 over a rolling window, and attach a notification channel. Keep the parentheses: `AND` binds tighter than `OR`. +### SigNoz + +Needs Docker Engine with the Compose plugin. Set `` to 12 or more characters, with upper and lower case, a number and a symbol. + +```bash +curl -fsSL https://signoz.io/foundry.sh | FOUNDRY_INSTALL_DIR=/usr/local/bin bash + +sed \ + -e "s/__POSTGRES_DB__/signoz/" -e "s/__POSTGRES_USER__/signoz/" \ + -e "s/__POSTGRES_PASSWORD__/$(openssl rand -hex 24)/" \ + -e "s/__SIGNOZ_PORT__/8090/" -e "s/__OTLP_GRPC_PORT__/4317/" -e "s/__OTLP_HTTP_PORT__/4318/" \ + -e "s/__SMTP_SMARTHOST__//" -e "s/__SMTP_USERNAME__//" -e "s/__SMTP_PASSWORD__//" -e "s/__SMTP_FROM__//" \ + -e "s|__SIGNOZ_EXTERNAL_URL__|http://localhost:8090|" \ + observability/signoz/casting.yaml.template > observability/signoz/casting.yaml -- **Publish failures**, adapter errors included: +foundryctl cast -f observability/signoz/casting.yaml +docker compose \ + -f pours/deployment/compose.yaml \ + -f observability/signoz/docker-compose.override.yml \ + up -d - ``` - service.name = 'relaysms-publisher-worker' AND (body CONTAINS 'Failed to process payload' OR body CONTAINS 'Failed to publish message' OR body CONTAINS 'An unexpected error occurred during task processing') - ``` +until curl -fs http://127.0.0.1:8090/api/v1/version | grep -q setupCompleted; do sleep 2; done +curl -X POST http://127.0.0.1:8090/api/v1/register \ + -H "Content-Type: application/json" \ + -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@relaysms.local","password":""}' +``` -- **Key management failures**: +The template stores SigNoz's metadata in Postgres (`metastore.kind: postgres`); set `sqlite` for a single node. + +### Uptime Kuma + +```bash +docker compose -f observability/uptime-kuma/docker-compose.yml up -d +``` + +### Reverse proxy + +```bash +sed -e "s/__SERVER_NAME__//g" \ + observability/nginx-observability.conf.template | + sudo tee /etc/nginx/sites-available/.conf >/dev/null +sudo ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ + +sed -e "s/__SERVER_NAME__//g" \ + observability/nginx-uptime-kuma.conf.template | + sudo tee /etc/nginx/sites-available/.conf >/dev/null +sudo ln -s /etc/nginx/sites-available/.conf /etc/nginx/sites-enabled/ + +sudo nginx -t && sudo systemctl reload nginx +sudo certbot --nginx -d +sudo certbot --nginx -d +``` - ``` - body CONTAINS 'Failed to generate server identity keys' OR body CONTAINS 'Token hash missing' - ``` +`` serves SigNoz and `` serves Uptime Kuma. diff --git a/docs/operating/services.md b/docs/operating/services.md index 9dfa11d..751773b 100644 --- a/docs/operating/services.md +++ b/docs/operating/services.md @@ -2,13 +2,30 @@ The Publisher runs as five systemd services, grouped under `relaysms-publisher.target`: -| Service | Runs | Listens on | -| --- | --- | --- | -| `relaysms-publisher-rest` | REST API | `HOST:PORT` (`127.0.0.1:16000`) | -| `relaysms-publisher-grpc` | gRPC API | `GRPC_HOST:GRPC_PORT` (`127.0.0.1:6000`) | -| `relaysms-publisher-worker` | Celery worker: publishes queued payloads | - | -| `relaysms-publisher-beat` | Celery Beat: schedules cleanups, and the worker heartbeat with Uptime Kuma | - | -| `relaysms-publisher-smtp` | [Email listener](../configuration/smtp-transport.md); exits at once while `SMTP_TRANSPORT_ENABLED` is off | - | +| Service | Runs | +| --- | --- | +| `relaysms-publisher-rest` | REST API | +| `relaysms-publisher-grpc` | gRPC API | +| `relaysms-publisher-worker` | Celery worker: publishes queued payloads | +| `relaysms-publisher-beat` | Celery Beat: schedules cleanups, and the worker heartbeat with Uptime Kuma | +| `relaysms-publisher-smtp` | [Email listener](../configuration/smtp-transport.md); exits at once while `SMTP_TRANSPORT_ENABLED` is off | + +The REST API listens on `HOST:PORT` (`127.0.0.1:16000`) and gRPC on `GRPC_HOST:GRPC_PORT` (`127.0.0.1:6000`). + +## Is it working? + +```bash +sudo ./manage.sh status +curl http://127.0.0.1:16000/health +``` + +You should see every service active, and `{"status":"ok"}`. To follow publications as they happen: + +```bash +sudo ./manage.sh logs --unit worker +``` + +Each published payload logs `Published via ''.`, and each failure logs `Failed to publish message`. Credentials with `stats:publications:read` can also query the counts at `GET /v1/stats/publications`. ## manage.sh @@ -24,9 +41,19 @@ Run it from the install directory with `sudo`: | `migrate` | Applies database migrations | | `check` | Reports every missing or invalid setting | | `nginx [DOMAIN]` | Renders the nginx site again and reattaches or obtains its certificate ([details](../installation/reverse-proxy.md#on-an-existing-install)) | -| `uninstall` | Removes the services and the install directory, with `.env` and `data/`. nginx sites, certificates, database and broker servers, and the service user stay. | +| `docs [DOMAIN]` | Builds this manual and serves it on its own domain ([details](../installation/server.md#serving-the-manual)) | +| `uninstall` | Removes the services and the install directory, with `.env` and `data/` | + +`uninstall` leaves the nginx sites, certificates, database and broker servers, and the service user. + +`logs` takes: + +- `-u`, `--unit`: `rest`, `grpc`, `worker`, `beat` or `smtp`; repeatable +- `-n`, `--lines`: how many lines to show +- `-s`, `--since`: a time, such as `"1 hour ago"` +- `--no-follow`: print and exit -`logs` takes `-u/--unit` (`rest`, `grpc`, `worker`, `beat` or `smtp`; repeatable), `-n/--lines` and `-s/--since`, and `--no-follow` prints and exits: +For example: ```bash sudo ./manage.sh logs --unit rest --unit grpc --since "1 hour ago" --lines 200 diff --git a/docs/platforms/bluesky.md b/docs/platforms/bluesky.md new file mode 100644 index 0000000..1f039ca --- /dev/null +++ b/docs/platforms/bluesky.md @@ -0,0 +1,67 @@ +# Bluesky + +Lets users post to Bluesky. Bluesky has no app registration: the Publisher serves the client's metadata at a public URL, and that URL is the client ID. + +*Last checked against the AT Protocol OAuth spec: 2026-10-22.* + +## Requirements + +- The REST API is public over HTTPS on port 443, behind the [reverse proxy](../installation/reverse-proxy.md). +- `https:///v1/platforms/bluesky/oauth/client-metadata.json` answers `200` with JSON, with no redirect on the way. It does while the adapter is enabled. +- The [redirect URIs](manage.md#redirect-uris) use HTTPS. + +## Configure the adapter + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/bluesky-oauth2-adapter +sudo ./publisher.sh platforms config set bluesky \ + client_id=https://publisher.example.com/v1/platforms/bluesky/oauth/client-metadata.json \ + client_uri=https://publisher.example.com \ + client_name=RelaySMS \ + redirect_uris=https://example.com/callback \ + "scope=atproto transition:generic" +``` + +| Setting | Required | Description | +| --- | --- | --- | +| `client_id` | Yes | The metadata URL above, exactly as the Publisher serves it | +| `client_uri` | Yes | Home page on the Publisher's host, the same host as `client_id` | +| `client_name` | Yes | Name on Bluesky's consent page | +| `redirect_uris` | Yes | Comma-separated redirect URIs your apps send, the default first | +| `scope` | Yes | `atproto transition:generic` | +| `pds_url` | No | Auth server. Default: `https://bsky.social` | + +## Check it + +```bash +curl -i https://publisher.example.com/v1/platforms/bluesky/oauth/client-metadata.json +``` + +You should see `200` and JSON whose `client_id` equals the URL you fetched. + +## Troubleshooting + +### Bluesky says the client metadata is invalid + +`client_id` must match the served URL exactly: `https`, `/v1`, no trailing slash. + +### The metadata URL redirects or needs a login + +Bluesky refuses redirects. Serve the URL directly over HTTPS. + +### `Access wasn't granted to every scope.` + +Set `scope` to exactly `atproto transition:generic`. + +### `The account uses a different auth server.` + +The account isn't on `pds_url`'s auth server, such as one on a self-hosted PDS. Only accounts there can link. + +### A changed redirect URI isn't accepted yet + +Bluesky's auth server caches the metadata. Try again later. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/platforms/gmail.md b/docs/platforms/gmail.md new file mode 100644 index 0000000..46db7d9 --- /dev/null +++ b/docs/platforms/gmail.md @@ -0,0 +1,99 @@ +# Gmail + +Lets users send email from their Gmail account. + +*Last checked against Google Cloud Console: 2026-10-22.* + +## Requirements + +- A Google Cloud account +- An HTTPS [redirect URI](manage.md#redirect-uris) + +## Create the OAuth client + +1. [Create a project](https://console.cloud.google.com/projectcreate), or pick one. +2. [Enable the Gmail API](https://console.cloud.google.com/apis/library/gmail.googleapis.com). +3. In Google Auth Platform > [Branding](https://console.cloud.google.com/auth/branding), click **Get started**. +4. Enter the app name, support email and contact email. Choose **External** as the audience, unless every user belongs to your Google Workspace organization. +5. In [Data Access](https://console.cloud.google.com/auth/scopes), add these scopes and save: + - `openid` + - `https://www.googleapis.com/auth/userinfo.email` + - `https://www.googleapis.com/auth/userinfo.profile` + - `https://www.googleapis.com/auth/gmail.send` +6. In [Clients](https://console.cloud.google.com/auth/clients), create a client of type **Web application**. +7. Under Authorized redirect URIs, add every redirect URI your apps send, the default first, and create the client. +8. Download the client's JSON from the dialog that confirms it. + +> [!IMPORTANT] +> Google shows the client secret only at creation. If you close the dialog without downloading the JSON, add a new secret to the client. + +## Configure the adapter + +Copy the JSON to the server, then: + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/gmail-oauth2-adapter +sudo ./publisher.sh platforms setup gmail \ + --input client_file=@/tmp/client_secret_.apps.googleusercontent.com.json +``` + +`setup` copies the client ID, secret and redirect URIs from the file. + +To add a redirect URI later, add it to the client in the console, then set the whole list: + +```bash +sudo ./publisher.sh platforms config set gmail \ + redirect_uris=https://a.example.com/callback,https://b.example.com/callback +``` + +## Check it + +```bash +sudo ./publisher.sh platforms config show gmail +``` + +You should see `redirect_uris` in the client's order, and `client_secret` as `(secret, set)`. + +## Testing and production + +| Publishing status | Who can link | Limits | +| --- | --- | --- | +| Testing | Test users listed under [Audience](https://console.cloud.google.com/auth/audience) | Links expire after 7 days | +| In production, unverified | Anyone, after a warning | 100 users in all | +| In production, verified | Anyone | None | + +`gmail.send` is a sensitive scope, so [verification](https://developers.google.com/identity/protocols/oauth2/production-readiness/sensitive-scope-verification) needs: + +- the redirect and home page domains verified in Google Search Console; +- a public home page and a privacy policy on that domain, describing how the app uses Gmail; +- a justification for each scope; +- an unlisted YouTube video showing the consent screen and how the app sends email. + +Publish the app under Audience, then submit it in the [Verification Center](https://console.cloud.google.com/auth/verification). + +## Troubleshooting + +### `setup` fails with `redirect_uris is required` + +The client has no authorized redirect URIs. Add them in the console, then set them with `config set gmail redirect_uris=...`. + +### `redirect_uri_mismatch` + +The URI the app sent isn't on the client. Add it exactly, trailing slash included. + +### Linking fails with `Access wasn't granted to ...` + +The user unticked a permission on the consent screen. They should link again and allow every permission. + +### Sending fails with `Google refused the refresh token` + +Links made while the app is in Testing expire after 7 days. Publish the app, and have users link again. + +### The client disappeared + +Google deletes clients unused for 6 months. Restore it within 30 days, or create a new one. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/platforms/manage.md b/docs/platforms/manage.md index 0fe4d61..016d777 100644 --- a/docs/platforms/manage.md +++ b/docs/platforms/manage.md @@ -6,18 +6,24 @@ Run these commands from the install directory with [`publisher.sh`](../operating ## Commands +| Command | Does | +| --- | --- | +| `platforms add [--tag ]` | Installs the newest version tag, or `` | +| `platforms update [] [--tag ]` | Moves one adapter, or all, to a version tag | +| `platforms list` | Lists every adapter, disabled ones included | +| `platforms disable ` | Hides it from users; it still revokes tokens | +| `platforms enable ` | Offers it again | +| `platforms remove [--force]` | Uninstalls it ([details](#removing)) | +| `platforms config show ` | Lists its settings, without secret values | +| `platforms config set =\|` | Changes settings; a bare `` asks for its value | +| `platforms setup [--input =]` | Runs the adapter's setup and saves its settings | +| `platforms import` | Registers adapter directories and moves their files out of the code | +| `platforms exec -- ` | Runs a command the adapter installs | + +Prefix each with `sudo ./publisher.sh`, for example: + ```bash -sudo ./publisher.sh platforms add [--tag TAG] # install the newest, or a given, version tag -sudo ./publisher.sh platforms update [NAME] [--tag TAG] # move one adapter, or all, to a version tag -sudo ./publisher.sh platforms list # every adapter, disabled ones included -sudo ./publisher.sh platforms disable # hide it from users; it still revokes tokens -sudo ./publisher.sh platforms enable # offer it again -sudo ./publisher.sh platforms remove [--force] # uninstall -sudo ./publisher.sh platforms config show # its settings, without secret values -sudo ./publisher.sh platforms config set KEY=VALUE|KEY # change settings; a bare KEY asks for the value -sudo ./publisher.sh platforms setup [--input KEY=VALUE] # run the adapter's setup and save its settings -sudo ./publisher.sh platforms import # register adapter directories, move their files out of the code -sudo ./publisher.sh platforms exec -- [ARGS] # run a command the adapter installs +sudo ./publisher.sh platforms add https://github.com/smswithoutborders/gmail-oauth2-adapter ``` Commands taking a name also take `--proto-id` and `--cat-id`, for when a name matches more than one adapter. Administrators can do the same over the [REST API](../reference/rest-api.md#managing-platform-adapters), but only from GitHub orgs listed in `PLATFORMS_GITHUB_ORGS`. @@ -26,26 +32,25 @@ Commands taking a name also take `--proto-id` and `--cat-id`, for when a name ma Adapters are installed and updated by version tag, such as `v1.2.0`, so a repository needs one before it can be added. -An update builds the new version and its virtualenv beside the running one, then switches over, so a failed update leaves the running version in place. - -> [!NOTE] -> An update is refused if the installed tag now points to a different commit. Publish a new tag instead of moving one. +An update builds the new version and its virtualenv beside the running one, then switches over, so a failed update leaves the running version in place. An update is refused if the installed tag now points to a different commit: publish a new tag instead of moving one. ## Removing +`remove` is refused while accounts are linked through the adapter, since their tokens can only be revoked through it. Disable the adapter instead. + > [!WARNING] -> `remove` is refused while accounts are linked through the adapter, since their tokens can only be revoked through it. Disable it instead. `--force` removes it anyway and leaves those tokens unrevocable. +> `--force` removes it anyway and leaves those tokens unrevocable. ## Files | Directory | Holds | -|---|---| -| `PLATFORMS_ADAPTERS_DIR/` | The adapter's code at its tag, replaced on every update | -| `PLATFORMS_ADAPTERS_VENV_DIR/` | Its virtualenv | -| `PLATFORMS_ADAPTERS_CONFIG_DIR/` | Its `credentials.json` | -| `PLATFORMS_ADAPTERS_STATE_DIR/` | What it writes, such as databases | +| --- | --- | +| `PLATFORMS_ADAPTERS_DIR/` | The adapter's code at its tag, replaced on every update | +| `PLATFORMS_ADAPTERS_VENV_DIR/` | Its virtualenv | +| `PLATFORMS_ADAPTERS_CONFIG_DIR/` | Its `credentials.json` | +| `PLATFORMS_ADAPTERS_STATE_DIR/` | What it writes, such as databases | -`` is the ID column of `platforms list`. +`` is the ID column of `platforms list`. ## Settings @@ -57,12 +62,33 @@ sudo ./publisher.sh platforms config set rmail smtp_password sudo ./publisher.sh platforms config set rmail --unset alias_prefix ``` -`config set` changes only the keys given. `setup` is for adapters whose settings come from the platform (e.g. a registered app for Mastodon, a downloaded client file for Gmail) and replaces all of them. `--input KEY=@PATH` reads a file: +`config set` changes only the keys given. `setup` is for adapters whose settings come from the platform, such as a registered app for Mastodon or a downloaded client file for Gmail, and replaces all of them. `--input =@` reads a file as the service user, from the install directory, so give an absolute path it can read: ```bash -sudo ./publisher.sh platforms setup gmail --input client_file=@client_secret.json +sudo ./publisher.sh platforms setup gmail --input client_file=@/tmp/client_secret.json ``` +Once an adapter is configured, link a test account from the RelaySMS app to check it end to end. + +## Redirect URIs + +OAuth2 adapters (Gmail, Mastodon, Bluesky) send the user back to a redirect URI after they authorize. The client app chooses it and sends it to the Publisher when it links an account; the adapter uses the first of its `redirect_uris` setting when the app sends none. + +1. Run an HTTPS page that receives the authorization `code` and `state` and hands them back to the app. +2. Configure the apps to send that page's URI. +3. Register the same URI, character for character, with the platform. A trailing slash counts. +4. Set it in the adapter's `redirect_uris`, with every other URI your apps send. + +## Provider guides + +| Platform | Adapter | Protocol | +| --- | --- | --- | +| [Gmail](gmail.md) | [gmail-oauth2-adapter](https://github.com/smswithoutborders/gmail-oauth2-adapter) | OAuth2 | +| [Mastodon](mastodon.md) | [mastodon-oauth2-adapter](https://github.com/smswithoutborders/mastodon-oauth2-adapter) | OAuth2 | +| [Bluesky](bluesky.md) | [bluesky-oauth2-adapter](https://github.com/smswithoutborders/bluesky-oauth2-adapter) | OAuth2 | +| [Telegram](telegram.md) | [telegram-pnba-adapter](https://github.com/smswithoutborders/telegram-pnba-adapter) | Phone number | +| [RelaySMS Mail](relaysms-mail.md) | [relaysms-mail-pnba-adapter](https://github.com/smswithoutborders/relaysms-mail-pnba-adapter) | Phone number | + ## An adapter's own commands An adapter may install admin commands. `exec` runs one from the adapter's virtualenv with its config and state directories; put `--` before it: diff --git a/docs/platforms/mastodon.md b/docs/platforms/mastodon.md new file mode 100644 index 0000000..c854e0c --- /dev/null +++ b/docs/platforms/mastodon.md @@ -0,0 +1,59 @@ +# Mastodon + +Lets users post to Mastodon. The adapter's setup registers the app on a Mastodon server, so there's no console to visit. + +*Last checked against the Mastodon API: 2026-10-22.* + +## Requirements + +- A Mastodon server running 4.4 or newer: see `version` at `https:///api/v2/instance` +- An HTTPS [redirect URI](manage.md#redirect-uris) + +The adapter serves one Mastodon server: only accounts on that server can link. + +## Configure the adapter + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/mastodon-oauth2-adapter +sudo ./publisher.sh platforms setup mastodon \ + --input client_name=RelaySMS \ + --input redirect_uris=https://example.com/callback \ + --input website=https://example.com \ + --input base_url=https://mastodon.social +``` + +| Input | Required | Description | +| --- | --- | --- | +| `client_name` | Yes | App name on the server's consent page | +| `redirect_uris` | Yes | Comma-separated redirect URIs your apps send, the default first | +| `website` | No | Home page on the consent page | +| `base_url` | No | The server. Default: `https://mastodon.social` | + +`setup` registers the app with the scopes `profile`, `write:statuses` and `write:media`, and saves its client ID and secret. Run it again to change the redirect URIs or the server; it registers a new app and replaces every setting. After changing the server, users must link again. + +## Check it + +```bash +sudo ./publisher.sh platforms config show mastodon +``` + +You should see `client_secret` as `(secret, set)`. + +## Troubleshooting + +### A user can't link + +Their account is on another server than `base_url`. + +### Linking fails after authorizing + +The server runs Mastodon older than 4.4. + +### "The redirect URI included is not valid" + +The app sent a URI the app registration doesn't have. Run `setup` again with it. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/platforms/relaysms-mail.md b/docs/platforms/relaysms-mail.md new file mode 100644 index 0000000..1cbfc70 --- /dev/null +++ b/docs/platforms/relaysms-mail.md @@ -0,0 +1,92 @@ +# RelaySMS Mail + +The `rmail` adapter gives each user an email alias tied to their phone number, and sends email from it through [SimpleLogin](https://simplelogin.io). Users link with a code sent over a messaging app by [Shortmesh Authy](https://github.com/shortmesh/Authy-API). It's the adapter that publishes [offline payloads](../configuration/offline-publishing.md), each from a new random alias. + +*Last checked against SimpleLogin, Shortmesh Authy and Google: 2026-10-22.* + +## Requirements + +- A SimpleLogin Premium account, which custom domains and sending to new contacts require +- A domain whose DNS you control +- An SMTP account allowed to send as your SimpleLogin mailbox address +- A self-hosted Shortmesh Authy, to send the login codes + +## SimpleLogin + +1. Sign up at [app.simplelogin.io](https://app.simplelogin.io). The signup address becomes your mailbox; to use another, add and verify it under [Mailboxes](https://app.simplelogin.io/dashboard/mailbox). +2. Add your domain, or a subdomain such as `mail.example.com`, under [Custom domains](https://app.simplelogin.io/dashboard/custom_domain). +3. Create the DNS records the domain's page lists: ownership TXT, MX, SPF, DKIM and DMARC. SimpleLogin marks the last three optional; without them, mail sent from aliases doesn't arrive. +4. Create an [API key](https://app.simplelogin.io/dashboard/api_key). + +## SMTP + +The adapter sends each message from your mailbox address over SMTP, and SimpleLogin relays it from the user's alias. Port 465 uses TLS; any other port uses STARTTLS when the server offers it. + +For a Gmail mailbox, use `smtp.gmail.com`, port `465`, the Gmail address, and an [app password](https://myaccount.google.com/apppasswords), which needs 2-Step Verification on the account. + +## Shortmesh Authy + +Run [Authy-API](https://github.com/shortmesh/Authy-API) with its Interface-API, then in the Interface-API's admin UI: + +1. Under Devices, link the phone that sends the codes. +2. Under Tokens, create a token. + +## Configure the adapter + +This example uses a Gmail mailbox: + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/relaysms-mail-pnba-adapter +sudo ./publisher.sh platforms config set rmail \ + sl_primary_email=you@gmail.com sl_primary_domain=mail.example.com \ + smtp_host=smtp.gmail.com smtp_port=465 smtp_username=you@gmail.com \ + authy_base_url=https://authy.example.com authy_sender=+237600000000 +sudo ./publisher.sh platforms config set rmail sl_api_key smtp_password authy_token +``` + +`sudo ./publisher.sh platforms config show rmail` lists the optional settings, such as alias prefixes. + +## Check it + +```bash +sudo ./publisher.sh platforms list --name rmail +``` + +You should see ✓ under Configured. + +```bash +curl https://authy.example.com/api/v1/platforms +``` + +You should see the channels Authy can send codes on, such as `signal`. + +## Troubleshooting + +### `No mailbox for ` + +`sl_primary_email` isn't a verified SimpleLogin mailbox. + +### `No suffix available for hostname ''` + +`sl_primary_domain` isn't a verified custom domain in SimpleLogin. + +### Sending fails with an SMTP authentication error + +For Gmail, use an app password, not the account password. A changed account password revokes app passwords. + +### Recipients don't get the email + +Check the domain's SPF, DKIM and DMARC records on its SimpleLogin page. + +### SimpleLogin refuses to create a contact + +The account isn't Premium. + +### Users don't get a login code + +Check that the sender device is linked and online, and that the channel the app sends, such as `signal`, appears at `/api/v1/platforms`. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/platforms/telegram.md b/docs/platforms/telegram.md new file mode 100644 index 0000000..420f4ab --- /dev/null +++ b/docs/platforms/telegram.md @@ -0,0 +1,49 @@ +# Telegram + +Lets users send Telegram messages. Users link with their phone number, a login code and, if they set one, their two-step verification password. The adapter needs a Telegram API ID and hash. + +*Last checked against my.telegram.org: 2026-10-22.* + +## Requirements + +- Agreement to Telegram's [API terms of service](https://core.telegram.org/api/terms) +- A Telegram account made with an official app, on a phone number you'll keep. Each number can create only one API ID. + +## Get the API ID and hash + +1. Sign in at [my.telegram.org](https://my.telegram.org) with the number in international format. The login code arrives in the Telegram app, not by SMS. +2. Open **API development tools**. +3. Fill in the app title and a short name (5 to 32 letters and digits), and create the application. +4. Copy the **App api_id** and **App api_hash**. + +The hash can't be changed: replacing a leaked one takes a new app on another number. + +## Configure the adapter + +```bash +sudo ./publisher.sh platforms add \ + https://github.com/smswithoutborders/telegram-pnba-adapter +sudo ./publisher.sh platforms config set telegram api_id= api_hash +``` + +## Check it + +```bash +sudo ./publisher.sh platforms list --name telegram +``` + +You should see ✓ under Configured. + +## Troubleshooting + +### The form at my.telegram.org says only "ERROR" + +Try again later from a home connection without a VPN, in a private window. + +### A user's account is limited or banned + +Heavy or spam-like sending gets accounts banned. Appeals go to recover@telegram.org. + +## Next steps + +[Register a gateway client](../gateway-clients/manage.md) to relay SMS to the Publisher. diff --git a/docs/reference/grpc-api.md b/docs/reference/grpc-api.md index 83d4293..ea17569 100644 --- a/docs/reference/grpc-api.md +++ b/docs/reference/grpc-api.md @@ -43,9 +43,9 @@ Generates an OAuth2 authorization URL for initiating the OAuth2 flow. | Field | Type | Required | Description | | :--- | :--- | :--- | :--- | -| platform | string | Yes | Platform identifier (e.g., "gmail") | -| state | string | Optional | State parameter to prevent CSRF attacks | -| code_verifier | string | Optional | Cryptographic random string for PKCE | +| platform | string | Yes | Platform identifier, such as `gmail` | +| state | string | Optional | OAuth2 `state`, returned unchanged | +| code_verifier | string | Optional | PKCE code verifier | | autogenerate_code_verifier | bool | Optional | Auto-generate code verifier if not provided | | redirect_url | string | Optional | Redirect URL for OAuth2 application | | request_identifier | string | Optional | Request identifier for tracking | @@ -76,7 +76,7 @@ grpcurl -plaintext \ "autogenerate_code_verifier": true }' \ -proto publisher.proto \ - : publisher.v3.Publisher/GetOAuth2AuthorizationUrl + : publisher.v3.Publisher/GetOAuth2AuthorizationUrl ``` ### Exchange OAuth2 code and store token @@ -107,7 +107,7 @@ Exchanges an OAuth2 authorization code for tokens and stores them. Returns encry | :--- | :--- | :--- | | success | bool | Operation success | | message | string | Response message | -| account_identifier | string | Account identifier (e.g., ) | +| account_identifier | string | Account identifier, such as `mail@example.com` | | token_ciphertext | bytes | Encrypted token ciphertext | | token_id | uint32 | Unique identifier for the stored token | | server_ephemeral_public_keys | list[PublicKey] | Exactly 256 ephemeral public keys generated by the server | @@ -123,7 +123,7 @@ grpcurl -plaintext \ -H 'x-timestamp: ' \ -d @ \ -proto publisher.proto \ - : publisher.v3.Publisher/ExchangeOAuth2CodeAndStore <: publisher.v3.Publisher/ExchangeOAuth2CodeAndStore <: publisher.v3.Publisher/RevokeOAuth2Token + : publisher.v3.Publisher/RevokeOAuth2Token ``` ### Get PNBA code @@ -182,7 +182,7 @@ Requests a one-time passcode (OTP) for Phone Number-Based Authentication. | platform | string | Yes | Platform identifier | | phone_number | string | Yes | Phone number to receive OTP | | request_identifier | string | Optional | Request identifier for tracking | -| channel | string | Optional | The medium used to send the code (e.g., "whatsapp", "telegram", "signal") | +| channel | string | Optional | Channel to send the code over, such as `whatsapp`, `telegram` or `signal` | **Response:** `GetPNBACodeResponse` @@ -206,7 +206,7 @@ grpcurl -plaintext \ "phone_number": "+1234567890" }' \ -proto publisher.proto \ - : publisher.v3.Publisher/GetPNBACode + : publisher.v3.Publisher/GetPNBACode ``` ### Exchange PNBA code and store token @@ -223,7 +223,7 @@ Exchanges a one-time passcode (OTP) for a token, stores it, and returns encrypte | password | string | Optional | Password for two-step verification | | request_identifier | string | Optional | Request identifier for tracking | | client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 ephemeral public keys generated by the client, with key_id 0-255 each used once | -| channel | string | Optional | The medium used to receive the code (e.g., "whatsapp", "telegram", "signal") | +| channel | string | Optional | Channel the code was sent over, such as `whatsapp`, `telegram` or `signal` | **Response:** `ExchangePNBACodeAndStoreResponse` @@ -249,7 +249,7 @@ grpcurl -plaintext \ -H 'x-timestamp: ' \ -d @ \ -proto publisher.proto \ - : publisher.v3.Publisher/ExchangePNBACodeAndStore <: publisher.v3.Publisher/ExchangePNBACodeAndStore <: publisher.v3.Publisher/RevokePNBAToken + : publisher.v3.Publisher/RevokePNBAToken ``` ### Sync keys @@ -329,7 +329,7 @@ grpcurl -plaintext \ -H 'x-timestamp: ' \ -d @ \ -proto publisher.proto \ - : publisher.v3.Publisher/SyncKeys <: publisher.v3.Publisher/SyncKeys </v1 Credentials and their scopes are managed with [`sudo ./publisher.sh creds`](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#credentials) or the `/v1/creds` endpoints. -* **Web session:** `POST /v1/auth/login`, then send the cookie with every request (`credentials: "include"` in `fetch`). Writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from this API's origin or `AUTH_WEB_ORIGINS`. Sessions end after `AUTH_SESSION_IDLE_MINUTES` idle (30) or `AUTH_SESSION_MAX_HOURS` (12). -* **HTTP Basic:** username and password on every request, e.g. `curl -u analyst: .../v1/stats/publications`. +- **Web session:** `POST /v1/auth/login`, then send the cookie with every request (`credentials: "include"` in `fetch`). Writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from this API's origin or `AUTH_WEB_ORIGINS`. Sessions end after `AUTH_SESSION_IDLE_MINUTES` idle (30) or `AUTH_SESSION_MAX_HOURS` (12). +- **HTTP Basic:** username and password on every request, such as `curl -u analyst: .../v1/stats/publications`. Scope changes apply on the credential's next request. Each endpoint's description names the scopes it needs. @@ -62,11 +62,11 @@ List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. | `gateway_clients.create`, `gateway_clients.update`, `gateway_clients.delete` | A gateway client changes. `details` has its fields, or what changed. | | `gateway_clients.enable`, `gateway_clients.disable` | A gateway client is listed publicly again, or hidden | -`outcome` is `success`, `denied` (the change went beyond the actor's scopes) or `failed` (a refused login for an existing username: wrong password or disabled). Successful HTTP Basic requests aren't recorded, since every request authenticates. `actor` is the username at the time, and null for the CLI or a failed login. Events are kept for `AUDIT_RETENTION_DAYS` (365 by default). +`outcome` is `success`, `denied` (the change went beyond the actor's scopes) or `failed` (a refused login for an existing username: wrong password or disabled). Successful HTTP Basic requests aren't recorded, since every request authenticates. `actor` is the username at the time, and null for the CLI or a failed login. Events are kept for `AUDIT_RETENTION_DAYS`. ## Publishing -`POST /v1/publications` takes JSON and tags payloads with protocol `https`. See [offline publishing](../configuration/offline-publishing.md) for how the protocol and `tag` decide whether offline payloads publish. +`POST /v1/publications` takes JSON and tags payloads with protocol `https`. It needs no credentials: a payload publishes only if it decrypts with the keys of a linked account or, for an offline payload, passes the [offline publishing](../configuration/offline-publishing.md) rules. See [offline publishing](../configuration/offline-publishing.md) for how the protocol and `tag` decide whether offline payloads publish. It only checks the payload's structure before queueing it. Decryption, payload type and adapter errors happen later, so they're logged on the server, not returned. diff --git a/docs/theme/custom.css b/docs/theme/custom.css new file mode 100644 index 0000000..ad14727 --- /dev/null +++ b/docs/theme/custom.css @@ -0,0 +1,4 @@ +/* Tables fill the column and keep long code readable. */ +.content table { width: 100%; } +.content td { vertical-align: top; } +.content td code { overflow-wrap: anywhere; } From 0f9d7ca911379a126a22cd85c9d44e49c3ae06ab Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Fri, 23 Oct 2026 11:14:26 +0100 Subject: [PATCH 11/13] docs: add the operating pages for credentials, upgrades, backups, audit, hardening and troubleshooting --- README.md | 15 +---- docs/SUMMARY.md | 12 ++-- docs/configuration/encryption-keys.md | 2 +- docs/configuration/settings.md | 2 +- docs/gateway-clients/manage.md | 6 +- docs/installation/docker.md | 7 +-- docs/openapi.json | 2 +- docs/operating/audit-log.md | 30 +++++++++ docs/operating/backup-restore.md | 89 +++++++++++++++++++++++++++ docs/operating/credentials.md | 54 ++++++++++++++++ docs/operating/security-hardening.md | 35 +++++++++++ docs/operating/services.md | 9 +-- docs/operating/troubleshooting.md | 84 +++++++++++++++++++++++++ docs/operating/upgrading.md | 80 ++++++++++++++++++++++++ docs/platforms/manage.md | 2 +- docs/reference/rest-api.md | 22 +------ publisher/api/rest/v1/audit.py | 2 +- 17 files changed, 392 insertions(+), 61 deletions(-) create mode 100644 docs/operating/audit-log.md create mode 100644 docs/operating/backup-restore.md create mode 100644 docs/operating/credentials.md create mode 100644 docs/operating/security-hardening.md create mode 100644 docs/operating/troubleshooting.md create mode 100644 docs/operating/upgrading.md diff --git a/README.md b/README.md index 4bd9348..08a911f 100644 --- a/README.md +++ b/README.md @@ -183,20 +183,7 @@ Gateway clients are the phone numbers that relay SMS to this server. `GET /v1/ga ## Credentials -Credentials log in to the REST API, each with its own scopes. One holding every scope is an administrator. They can also be managed over the [REST API](docs/reference/rest-api.md#managing-credentials). - -```bash -sudo ./publisher.sh creds scopes # list scopes -sudo ./publisher.sh creds create --username ops --administrator # prints the password once -sudo ./publisher.sh creds create --username analyst --scope stats:publications:read -sudo ./publisher.sh creds set-scopes --username analyst --scope stats:publications:read --scope stats:publications:reasons -sudo ./publisher.sh creds list -sudo ./publisher.sh creds reset-password --username analyst -sudo ./publisher.sh creds disable --username analyst -sudo ./publisher.sh creds enable --username analyst -sudo ./publisher.sh creds revoke-sessions --username analyst -sudo ./publisher.sh creds delete --username analyst -``` +Credentials log in to the REST API's admin endpoints, each with its own scopes. See [Credentials and access](docs/operating/credentials.md). ## Documentation diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 5f4e49e..fb63c38 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -41,13 +41,13 @@ # Operating - [Services](operating/services.md) -- [Credentials and access]() -- [Upgrading]() -- [Backup and restore]() +- [Credentials and access](operating/credentials.md) +- [Upgrading](operating/upgrading.md) +- [Backup and restore](operating/backup-restore.md) - [Observability](operating/observability.md) -- [Audit log]() -- [Security hardening]() -- [Troubleshooting]() +- [Audit log](operating/audit-log.md) +- [Security hardening](operating/security-hardening.md) +- [Troubleshooting](operating/troubleshooting.md) # Reference diff --git a/docs/configuration/encryption-keys.md b/docs/configuration/encryption-keys.md index 7382a2e..c80eaf2 100644 --- a/docs/configuration/encryption-keys.md +++ b/docs/configuration/encryption-keys.md @@ -11,4 +11,4 @@ Three keys in `.env` protect the data. The installer and `make .env` generate th `template.env` turns both on with `DATABASE_FIELD_ENCRYPTION_ENABLED=true` and `DATABASE_ENCRYPTION_ENABLED=true`. > [!CAUTION] -> Back up `.env` with the database, and never change a key once the install holds data. Data encrypted with a lost or changed key can't be read again, and a new `DATA_ENCRYPTION_KEY` also gives every sender a new hash. +> Back up `.env` whenever you back up the database, store it apart ([Backup and restore](../operating/backup-restore.md)), and never change a key once the install holds data. Data encrypted with a lost or changed key can't be read again, and a new `DATA_ENCRYPTION_KEY` also gives every sender a new hash. diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 0130f1d..4b2527b 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -115,7 +115,7 @@ With `PLATFORMS_GITHUB_ORGS` empty, the REST API refuses to install or update ad ## REST API access -See [REST API](../reference/rest-api.md#authentication). +See [Credentials and access](../operating/credentials.md#logging-in). | Setting | Default | Description | | --- | --- | --- | diff --git a/docs/gateway-clients/manage.md b/docs/gateway-clients/manage.md index 5934f95..3d2ae8a 100644 --- a/docs/gateway-clients/manage.md +++ b/docs/gateway-clients/manage.md @@ -1,6 +1,6 @@ # Managing gateway clients -A gateway client is a phone number that relays the SMS people send to this server: an Android phone running Deku SMS, or a Twilio number. Register each number here so the RelaySMS app lists it, then [connect its relay](connect-a-relay.md). The enabled ones are listed publicly at `GET /v1/gateway-clients`. Credentials with `gc:read` and `gc:write` can manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients). Every change is recorded in the [audit log](../reference/rest-api.md#audit-events). +A gateway client is a phone number that relays the SMS people send to this server: an Android phone running Deku SMS, or a Twilio number. Register each number here so the RelaySMS app lists it, then [connect its relay](connect-a-relay.md). The enabled ones are listed publicly at `GET /v1/gateway-clients`. Credentials with `gc:read` and `gc:write` can manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients). Every change is recorded in the [audit log](../operating/audit-log.md). ## Commands @@ -38,7 +38,3 @@ Names are compared word by word, ignoring case, accents, company-type words ("Te | `none` | None: the number couldn't be placed. Enter the details yourself. | A PLMN missing from the snapshot can still be entered with `--operator-code`. - -## Upgrading from the JSON registry - -Migration 019 imports `data/gateway_clients/registry.json` (or `GATEWAY_CLIENTS_REGISTRY_FILE`, if `.env` sets it). It logs any entries in `mcc_mnc_overrides.json` without importing them. Both files can then be deleted. diff --git a/docs/installation/docker.md b/docs/installation/docker.md index cf2d12c..b7cbb5c 100644 --- a/docs/installation/docker.md +++ b/docs/installation/docker.md @@ -15,13 +15,10 @@ The container applies pending migrations, then starts every service. It publishe Run the admin CLI with `docker compose exec publisher python -m publisher `. -Back up `.env` with `./data`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. +[Back up](../operating/backup-restore.md) `.env` and `./data`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. ## Updating -```bash -git pull --recurse-submodules -docker compose up -d --build -``` +See [Upgrading](../operating/upgrading.md#update-with-docker). Next: [Settings reference](../configuration/settings.md). diff --git a/docs/openapi.json b/docs/openapi.json index 2686bb2..b659ca3 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -848,7 +848,7 @@ "Audit" ], "summary": "List audit events", - "description": "Newest first. Scope: audit:read, plus creds:read or platforms:read per area.", + "description": "Newest first. Scope: audit:read, plus each area's read scope.", "operationId": "list_audit_events_v1_audit_events_get", "security": [ { diff --git a/docs/operating/audit-log.md b/docs/operating/audit-log.md new file mode 100644 index 0000000..d41db03 --- /dev/null +++ b/docs/operating/audit-log.md @@ -0,0 +1,30 @@ +# Audit log + +The Publisher records every change to credentials, adapters and gateway clients, every web login and every refused login, whether made over the REST API or the CLI. Read it at `GET /v1/audit-events`: + +```bash +curl -u ops: https:///v1/audit-events +``` + +It needs `audit:read`, plus the read scope of each area: `creds:read` for `auth.*` and `creds.*` events, `platforms:read` for `platforms.*`, `gc:read` for `gateway_clients.*`. Events from areas you can't read are left out. Filter with `action`, `actor`, `target`, `since` and `until`, such as `?action=auth.login&since=2026-10-01`. Results are [paginated](../reference/rest-api.md#pagination). + +## Events + +| Action | Recorded when | +| --- | --- | +| `creds.create`, `creds.update`, `creds.reset_password`, `creds.revoke_sessions`, `creds.delete` | A credential changes. `details` says what changed, never secrets. | +| `auth.login` | A web session starts, or a web or HTTP Basic login for an existing username is refused | +| `auth.logout` | A web session ends | +| `platforms.add`, `platforms.update`, `platforms.remove` | An adapter is installed, updated or removed. `details` has the source URL and commits. | +| `platforms.enable`, `platforms.disable` | An adapter is offered to users again, or hidden from them | +| `platforms.configure`, `platforms.setup` | An adapter's settings change. `details` has the keys saved. | +| `gateway_clients.create`, `gateway_clients.update`, `gateway_clients.delete` | A gateway client changes. `details` has its fields, or what changed. | +| `gateway_clients.enable`, `gateway_clients.disable` | A gateway client is listed publicly again, or hidden | + +## Fields + +- `outcome`: `success`, `denied` (the change went beyond the actor's scopes) or `failed` (a refused login for an existing username: wrong password, or disabled). +- `actor`: the username at the time; null for the CLI or a failed login. +- `details`: what changed, never secrets; for `auth.login`, `method` is `session` or `basic`. + +Successful HTTP Basic requests aren't recorded. Events are kept for `AUDIT_RETENTION_DAYS` ([Retention](../configuration/settings.md#retention)). diff --git a/docs/operating/backup-restore.md b/docs/operating/backup-restore.md new file mode 100644 index 0000000..1cc5316 --- /dev/null +++ b/docs/operating/backup-restore.md @@ -0,0 +1,89 @@ +# Backup and restore + +A backup is: + +- **`.env`**: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. +- **`data/`**: the SQLite database, Celery files, and the adapters with their settings, state and virtualenvs. +- **A dump of the database**, with MySQL or PostgreSQL. +- **Any file `.env` points to outside `data/`**, such as `GRPC_TLS_CERT_FILE` or `SMTP_TLS_CLIENT_KEY`. + +The installer recreates the nginx sites, certificates and service units, so `/etc` needs no backup. + +> [!CAUTION] +> Store the `.env` archive apart from the data. Anyone with both can read the linked accounts' tokens. + +## Back up + +From the install directory, stop the services so the data is consistent, then archive `.env` and `data/` separately: + +```bash +sudo ./manage.sh stop +sudo tar -czf /root/relaysms-env-$(date +%F).tar.gz .env +sudo tar -czf /root/relaysms-data-$(date +%F).tar.gz data +``` + +With PostgreSQL, dump the database before starting the services again: + +```bash +sudo sh -c 'sudo -u postgres pg_dump relaysms > /root/relaysms-db-$(date +%F).sql' +``` + +With MySQL or MariaDB: + +```bash +sudo sh -c 'mysqldump relaysms > /root/relaysms-db-$(date +%F).sql' +``` + +Then start the services: + +```bash +sudo ./manage.sh start +``` + +For a database on another host, or named differently in `.env`, connect with the details from `.env`, such as `pg_dump -h -U `. With Docker, use `docker compose stop` and `docker compose start` in place of `manage.sh`. + +## Restore + +1. [Install](../installation/server.md) the Publisher in the same directory, with the same `--instance-name`. With MySQL or PostgreSQL, pass `--setup-db`, `--db-name`, `--db-user` and `--db-password` with the values from the backed-up `.env` (`MYSQL_*` or `POSTGRES_*`); with RabbitMQ, the same for `--broker-password`. Otherwise the servers refuse the restored `.env`'s logins. +2. Stop the services: + + ```bash + sudo ./manage.sh stop + ``` + +3. From the install directory, replace the new `data/` and `.env` with the backed-up ones: + + ```bash + sudo rm -rf data + sudo tar -xzf /root/relaysms-env-.tar.gz + sudo tar -xzf /root/relaysms-data-.tar.gz + ``` + +4. With PostgreSQL, replace the new database with the dump: + + ```bash + sudo -u postgres dropdb relaysms + sudo -u postgres createdb -O relaysms relaysms + sudo sh -c 'sudo -u postgres psql -v ON_ERROR_STOP=1 relaysms < /root/relaysms-db-.sql' + ``` + + With MySQL or MariaDB: + + ```bash + sudo mysql -e 'DROP DATABASE relaysms; CREATE DATABASE relaysms CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' + sudo sh -c 'mysql relaysms < /root/relaysms-db-.sql' + ``` + +5. Apply any migrations newer than the backup, then start the services and check them: + + ```bash + sudo ./manage.sh migrate + sudo ./manage.sh start + sudo ./manage.sh check + ``` + +`tar` keeps the files' owners. If the service user's name or ID differs on the new host, give the files back to it: `sudo chown -R : data` and `sudo chown root: .env`. + +## Next steps + +[Check that the services are up](services.md#is-it-working). diff --git a/docs/operating/credentials.md b/docs/operating/credentials.md new file mode 100644 index 0000000..1ded97b --- /dev/null +++ b/docs/operating/credentials.md @@ -0,0 +1,54 @@ +# Credentials and access + +Credentials log in to the REST API's admin endpoints, each with its own scopes. One holding every scope is an administrator. The RelaySMS app and gateway clients need none. + +## Create a credential + +```bash +sudo ./publisher.sh creds create --username ops --administrator +sudo ./publisher.sh creds create --username analyst --scope stats:publications:read +``` + +> [!IMPORTANT] +> The password is printed only once. + +Usernames are 3 to 32 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit. + +## Scopes + +| Scope | Allows | +| --- | --- | +| `stats:publications:read` | Read publication stats, without failure reasons | +| `stats:publications:reasons` | See failure reasons in publication stats | +| `gc:read` | List gateway clients | +| `gc:write` | Add, change and remove gateway clients | +| `platforms:read` | List platform adapters | +| `platforms:write` | Change and remove platform adapters | +| `creds:read` | List credentials | +| `creds:write` | Add, change and remove credentials | +| `audit:read` | Read the [audit log](audit-log.md), for the areas your other scopes cover | + +`sudo ./publisher.sh creds scopes` prints the same list. Each `:write` scope needs its `:read` scope, and `stats:publications:reasons` needs `stats:publications:read`. Installing, updating and configuring adapters over the API also needs an administrator. + +## Commands + +| Command | Does | +| --- | --- | +| `creds list` | Lists credentials | +| `creds set-scopes --username --scope ` | Replaces a credential's scopes; repeat `--scope`, or pass `--administrator` | +| `creds reset-password --username ` | Prints a new password and ends the credential's web sessions | +| `creds disable --username ` | Blocks it and ends its web sessions | +| `creds enable --username ` | Lets it log in again | +| `creds revoke-sessions --username ` | Logs it out of every web session; HTTP Basic still works | +| `creds delete --username ` | Deletes it and its sessions, after asking; `--yes` skips the question | + +Prefix each with `sudo ./publisher.sh`. Credentials with `creds:write` can do the same over the [REST API](../reference/rest-api.md#managing-credentials), within their own scopes. + +## Logging in + +- **Web session:** `POST /v1/auth/login`, then send the session cookie with every request (`credentials: "include"` in `fetch`). Logins and writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from the API's own origin or one listed in `AUTH_WEB_ORIGINS`, on the same site as the API. A session ends after `AUTH_SESSION_IDLE_MINUTES` idle (30) or `AUTH_SESSION_MAX_HOURS` (12). +- **HTTP Basic:** the username and password on every request, such as `curl -u analyst: https:///v1/stats/publications`. + +Scope changes apply on the credential's next request. nginx allows 5 logins a minute per IP address, and 30 HTTP Basic requests a minute per IP address to `/v1/stats`, `/v1/auth` and `/v1/creds`. + +Settings: [REST API access](../configuration/settings.md#rest-api-access). diff --git a/docs/operating/security-hardening.md b/docs/operating/security-hardening.md new file mode 100644 index 0000000..39721d2 --- /dev/null +++ b/docs/operating/security-hardening.md @@ -0,0 +1,35 @@ +# Security hardening + +A server install listens on `127.0.0.1` only, runs each service under systemd sandboxing with everything read-only except its data directories, and sits behind nginx with TLS and rate limits. These steps close what's left to you. + +## Network + +1. Allow only SSH, HTTP and HTTPS in: + + ```bash + sudo ufw allow OpenSSH + sudo ufw allow 'Nginx Full' + sudo ufw enable + ``` + +2. Keep `HOST` and `GRPC_HOST` at `127.0.0.1`, so only nginx reaches the services. +3. On Docker, bind `PORT` and `GRPC_PORT` to `127.0.0.1` in `docker-compose.yml`, since Docker's published ports bypass ufw, and put nginx in front ([details](../installation/docker.md)). + +## Service user + +The services run as the user who ran the installer with `sudo`, or as a new `relaysms` user when it ran from a root shell. For a dedicated user with no login or `sudo` rights, run the installer from a root shell (`sudo -i`). Changing it on an existing install means reinstalling. + +## Settings + +| Setting | Keep it at | +| --- | --- | +| `API_DOCS_ENABLED` | `false` in production | +| `AUTH_SESSION_COOKIE_SECURE` | `true` | +| `AUTH_WEB_ORIGINS` | Only the web client origins you run | +| `PLATFORMS_GITHUB_ORGS` | Only orgs you trust: installing an adapter runs its code on the server | +| `SMTP_ALLOWED_SENDERS` | Exact addresses; a domain trusts every sender on it ([details](../configuration/smtp-transport.md#allowlist)) | +| `DATABASE_ENCRYPTION_ENABLED`, `DATABASE_FIELD_ENCRYPTION_ENABLED` | `true` | + +## Credentials + +Give each person their own [credential](credentials.md), with only the scopes they need. Review the [audit log](audit-log.md) for `denied` events (a change beyond the actor's scopes) and `failed` ones (a refused login). diff --git a/docs/operating/services.md b/docs/operating/services.md index 751773b..d52a71f 100644 --- a/docs/operating/services.md +++ b/docs/operating/services.md @@ -25,7 +25,7 @@ You should see every service active, and `{"status":"ok"}`. To follow publicatio sudo ./manage.sh logs --unit worker ``` -Each published payload logs `Published via ''.`, and each failure logs `Failed to publish message`. Credentials with `stats:publications:read` can also query the counts at `GET /v1/stats/publications`. +Each published payload logs `Published via ''.`, and each failure a line starting `Failed to` ([what each means](troubleshooting.md#a-payload-is-accepted-but-never-published)). Credentials with `stats:publications:read` can also query the counts at `GET /v1/stats/publications`. ## manage.sh @@ -37,7 +37,7 @@ Run it from the install directory with `sudo`: | `logs` | Follows the service logs | | `start`, `stop`, `restart` | Controls every service together | | `enable`, `disable` | Starts the services on boot, or stops doing so | -| `update` | Pulls the latest code, rebuilds and restarts the services | +| `update [--migrate]` | Pulls the latest code, rebuilds and restarts the services ([details](upgrading.md)) | | `migrate` | Applies database migrations | | `check` | Reports every missing or invalid setting | | `nginx [DOMAIN]` | Renders the nginx site again and reattaches or obtains its certificate ([details](../installation/reverse-proxy.md#on-an-existing-install)) | @@ -59,10 +59,7 @@ For example: sudo ./manage.sh logs --unit rest --unit grpc --since "1 hour ago" --lines 200 ``` -> [!IMPORTANT] -> Run `update --migrate` to apply the release's database migrations; with none pending it changes nothing. They run after the dependencies are reinstalled and before the services restart. - -`update` also checks `.env`. When a setting is invalid, it still restarts every service, then exits with an error; services that use that setting fail to start. +When `.env` has an invalid setting, `update` still restarts every service, then exits with an error; services that use that setting fail to start ([fix](troubleshooting.md#update-ends-with-env-has-errors)). See [Upgrading](upgrading.md). ## publisher.sh diff --git a/docs/operating/troubleshooting.md b/docs/operating/troubleshooting.md new file mode 100644 index 0000000..e0e889c --- /dev/null +++ b/docs/operating/troubleshooting.md @@ -0,0 +1,84 @@ +# Troubleshooting + +Start with the logs of the service involved: + +```bash +sudo ./manage.sh logs --unit --since "1 hour ago" +``` + +`` is `rest`, `grpc`, `worker`, `beat` or `smtp`. + +## Services + +### A service won't start + +The service refuses to start on an invalid setting it uses. Run `sudo ./manage.sh check`, fix what it reports in `.env`, then `sudo ./manage.sh restart`. + +### `update` ends with `.env has errors` + +The new version needs a setting `.env` lacks. Run `sudo ./sync-env.sh`, fill in what `sudo ./manage.sh check` reports, then `sudo ./manage.sh restart`. + +### `/health` returns `500` + +The REST API can't open the database: the database server is down, its settings in `.env` are wrong, or, with SQLite, the service user can't read `data/`. See [Database](../installation/database.md). + +### The email listener service is inactive + +It exits at once while `SMTP_TRANSPORT_ENABLED` is off. See [SMTP transport](../configuration/smtp-transport.md). + +### Docker refuses connections right after starting + +The container applies migrations before the services start. Wait, or follow `docker compose logs -f`. + +## Publishing + +### A payload is accepted but never published + +`POST /v1/publications` only queues a payload; the worker decides later. Follow `sudo ./manage.sh logs --unit worker` and match the line: + +| Log line | Cause | +| --- | --- | +| `Published via ''.` | It worked | +| `Failed to process payload: ...` | The payload didn't decrypt or validate, its keys are unknown or used up, or offline publishing refused it ([Offline publishing](../configuration/offline-publishing.md)) | +| `Failed to publish message: Adapter error: ...` | The platform refused it; the rest of the line is the adapter's reason | +| `Session can't be assembled from segment(s): ...` | A multi-part payload's segments don't fit together | +| `An unexpected error occurred during task processing.` | A bug; the traceback follows. [Report it](https://github.com/smswithoutborders/RelaySMS-Publisher/issues). | +| No line at all | The worker isn't running (`sudo ./manage.sh status`), or a multi-part payload is still waiting for its other segments, for up to `PAYLOAD_SESSION_MAX_AGE_HOURS` | + +`GET /v1/stats/publications` lists each failure with a reason code, for credentials with `stats:publications:reasons`. + +### Linking an account fails + +Check the adapter shows ✓ under Configured in `sudo ./publisher.sh platforms list`, then see its [provider guide](../platforms/manage.md#provider-guides)'s troubleshooting. + +## REST API + +### `Invalid username or password.` + +The credential doesn't exist, the password is wrong, or it's disabled (`sudo ./publisher.sh creds list`). + +### `Missing scope: ...` + +Add the scope with `sudo ./publisher.sh creds set-scopes` ([Scopes](credentials.md#scopes)). + +### `Session expired or invalid. Please log in again.` + +The session idled past `AUTH_SESSION_IDLE_MINUTES`, reached `AUTH_SESSION_MAX_HOURS`, or was ended by `reset-password`, `disable` or `revoke-sessions`. Log in again. + +### `Origin not allowed.` + +The web client's origin is neither the API's own nor listed in `AUTH_WEB_ORIGINS`; logins and writes with a session cookie both check it. Add the origin, which must be on the same site as the API, then `sudo ./manage.sh restart`. + +### `Too many requests. Please try again shortly.` + +nginx allows 5 logins a minute per IP address, and 30 HTTP Basic requests a minute per IP address to `/v1/stats`, `/v1/auth` and `/v1/creds`. Wait a minute and try again. + +## nginx + +### gRPC calls fail through nginx + +gRPC needs HTTP/2 on the HTTPS block. Run `sudo ./manage.sh nginx` to render the site again with it. + +### `nginx -t` fails after an update + +Run `sudo ./manage.sh nginx`: it renders the site from the current template, and restores the previous file if the new one fails. diff --git a/docs/operating/upgrading.md b/docs/operating/upgrading.md new file mode 100644 index 0000000..3ebee08 --- /dev/null +++ b/docs/operating/upgrading.md @@ -0,0 +1,80 @@ +# Upgrading + +A server install follows the branch it was installed from, `main` by default. An update pulls that branch, rebuilds, and restarts the services. + +## Before you upgrade + +1. [Back up](backup-restore.md). Migrations can't be undone, so going back to older code needs the backup. +2. Note the current commit, to roll back to: + + ```bash + git rev-parse HEAD + ``` + +## Update a server install + +From the install directory: + +```bash +sudo ./manage.sh update --migrate +sudo ./sync-env.sh +sudo ./manage.sh check +sudo ./manage.sh restart +``` + +1. `update --migrate` stops the services, pulls the code, reinstalls the dependencies, rebuilds, applies pending migrations and starts the services again. With no migrations pending, `--migrate` changes nothing. When the install serves this manual, it rebuilds it too. +2. `sync-env.sh` adds the settings the new `template.env` has and `.env` lacks, showing the change before writing it. +3. `check` reports any setting still missing or invalid; fill it in `.env`. +4. `restart` loads the new settings. + +If `git diff --stat ORIG_HEAD HEAD -- deploy/nginx` lists changes, render the site again: + +```bash +sudo ./manage.sh nginx +``` + +If you edited tracked files that the update changes, `update` stops before pulling and leaves the services stopped. Discard the edits or `git stash` them, then run `update` again. + +## Roll back + +1. Take the commit you noted [before upgrading](#before-you-upgrade). +2. If the update applied migrations, [restore the backup](backup-restore.md#restore) taken before it; older code can't use the migrated database. +3. Check out that commit and rebuild. `--pulled` skips the pull that would undo the checkout: + + ```bash + sudo git checkout + sudo ./manage.sh update --pulled + ``` + +To follow the branch again later, run `sudo git checkout main`, then `sudo ./manage.sh update --migrate`. + +## Update with Docker + +```bash +git pull --recurse-submodules +docker compose up -d --build +``` + +The container applies new migrations and runs `platforms import` when it starts. + +## Update adapters + +Adapters update separately, by version tag: + +```bash +sudo ./publisher.sh platforms update +``` + +See [Versions](../platforms/manage.md#versions). + +## Upgrade notes + +Steps a particular upgrade needs once, newest first. + +### Gateway clients moved to the database + +`update --migrate` applies migration 019, which imports `data/gateway_clients/registry.json` (or `GATEWAY_CLIENTS_REGISTRY_FILE`, if `.env` sets it). It logs the entries in `mcc_mnc_overrides.json` without importing them. Delete both files afterwards. + +### Adapters moved to the database + +`update --migrate` and `manage.sh migrate` register adapters installed before the adapter registry moved to the database, and move their settings and files out of the code. The Docker container does it at every start. diff --git a/docs/platforms/manage.md b/docs/platforms/manage.md index 016d777..d264f5e 100644 --- a/docs/platforms/manage.md +++ b/docs/platforms/manage.md @@ -17,7 +17,7 @@ Run these commands from the install directory with [`publisher.sh`](../operating | `platforms config show ` | Lists its settings, without secret values | | `platforms config set =\|` | Changes settings; a bare `` asks for its value | | `platforms setup [--input =]` | Runs the adapter's setup and saves its settings | -| `platforms import` | Registers adapter directories and moves their files out of the code | +| `platforms import` | Registers adapters installed before the database registry ([details](../operating/upgrading.md#adapters-moved-to-the-database)) | | `platforms exec -- ` | Runs a command the adapter installs | Prefix each with `sudo ./publisher.sh`, for example: diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index f64c652..3c4edf7 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -16,12 +16,7 @@ https:///v1 ## Authentication -Credentials and their scopes are managed with [`sudo ./publisher.sh creds`](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/README.md#credentials) or the `/v1/creds` endpoints. - -- **Web session:** `POST /v1/auth/login`, then send the cookie with every request (`credentials: "include"` in `fetch`). Writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from this API's origin or `AUTH_WEB_ORIGINS`. Sessions end after `AUTH_SESSION_IDLE_MINUTES` idle (30) or `AUTH_SESSION_MAX_HOURS` (12). -- **HTTP Basic:** username and password on every request, such as `curl -u analyst: .../v1/stats/publications`. - -Scope changes apply on the credential's next request. Each endpoint's description names the scopes it needs. +Admin endpoints take a web session cookie or HTTP Basic credentials; see [Logging in](../operating/credentials.md#logging-in). Each endpoint's description names the scopes it needs. ## Managing credentials @@ -49,20 +44,7 @@ List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. ## Audit events -`GET /v1/audit-events` needs `audit:read`, and each area also needs its read scope: `creds:read` for `auth.*` and `creds.*` events, `platforms:read` for `platforms.*`, `gc:read` for `gateway_clients.*`. Events from areas you can't read are left out. - -| Action | Recorded when | -| :--- | :--- | -| `creds.create`, `creds.update`, `creds.reset_password`, `creds.revoke_sessions`, `creds.delete` | A credential changes, over the API or the CLI. `details` says what changed, never secrets. | -| `auth.login` | A web session starts, or a login for an existing username is refused | -| `auth.logout` | A web session ends | -| `platforms.add`, `platforms.update`, `platforms.remove` | An adapter is installed, updated or removed. `details` has the source URL and commits. | -| `platforms.enable`, `platforms.disable` | An adapter is offered to users again, or hidden from them | -| `platforms.configure`, `platforms.setup` | An adapter's settings are changed, or set up from inputs. `details` has the keys saved. | -| `gateway_clients.create`, `gateway_clients.update`, `gateway_clients.delete` | A gateway client changes. `details` has its fields, or what changed. | -| `gateway_clients.enable`, `gateway_clients.disable` | A gateway client is listed publicly again, or hidden | - -`outcome` is `success`, `denied` (the change went beyond the actor's scopes) or `failed` (a refused login for an existing username: wrong password or disabled). Successful HTTP Basic requests aren't recorded, since every request authenticates. `actor` is the username at the time, and null for the CLI or a failed login. Events are kept for `AUDIT_RETENTION_DAYS`. +`GET /v1/audit-events` returns the [audit log](../operating/audit-log.md). ## Publishing diff --git a/publisher/api/rest/v1/audit.py b/publisher/api/rest/v1/audit.py index e80a2f1..5856ca7 100644 --- a/publisher/api/rest/v1/audit.py +++ b/publisher/api/rest/v1/audit.py @@ -46,7 +46,7 @@ def list_audit_events( ), db: Session = Depends(get_db), ) -> AuditEventPage: - """Newest first. Scope: audit:read, plus creds:read or platforms:read per area.""" + """Newest first. Scope: audit:read, plus each area's read scope.""" since, until = check_time_range(since, until) page = audit_event.list_events( db, From 74b2fa91daf7971b9cea5cce1cc988c497362223 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Fri, 23 Oct 2026 12:03:28 +0100 Subject: [PATCH 12/13] docs: add the reference, explanation and developer pages --- docs/README.md | 6 ++- docs/SUMMARY.md | 12 ++--- docs/developers/writing-an-adapter.md | 30 ++++++++++++ docs/explanation/architecture.md | 67 +++++++++++++++++++++++++++ docs/explanation/data-retention.md | 19 ++++++++ docs/explanation/security-model.md | 38 +++++++++++++++ docs/platforms/manage.md | 2 +- docs/reference/cli.md | 31 +++++++++++++ docs/reference/glossary.md | 24 ++++++++++ tests/cli/test_reference.py | 45 ++++++++++++++++++ 10 files changed, 265 insertions(+), 9 deletions(-) create mode 100644 docs/developers/writing-an-adapter.md create mode 100644 docs/explanation/architecture.md create mode 100644 docs/explanation/data-retention.md create mode 100644 docs/explanation/security-model.md create mode 100644 docs/reference/cli.md create mode 100644 docs/reference/glossary.md create mode 100644 tests/cli/test_reference.py diff --git a/docs/README.md b/docs/README.md index 6124703..75335ba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -44,11 +44,13 @@ The shaded box is the Publisher. ## Key terms - **Gateway client**: a phone number that relays SMS to the Publisher, from an Android phone running Deku SMS or from Twilio. See [Connecting a relay](gateway-clients/connect-a-relay.md). -- **Platform adapter**: the plugin that publishes to one platform, such as Gmail, installed from its own repository. See [Managing adapters](platforms/manage.md). +- **Adapter** (platform adapter): the plugin that publishes to one platform, such as Gmail, installed from its own repository. See [Managing adapters](platforms/manage.md). - **Linked account**: a user's account on a platform, authorized once while online. The Publisher stores its token encrypted. - **OAuth2 and PNBA**: the two ways an account links. OAuth2 goes through the platform's consent page (Gmail, Mastodon, Bluesky). Phone number-based authentication (PNBA) uses a code sent to the user's phone (Telegram, RelaySMS Mail). - **Offline payload**: content sent without a linked account, published through [RelaySMS Mail](platforms/relaysms-mail.md). +The [glossary](reference/glossary.md) defines the rest, and [Architecture](explanation/architecture.md) shows every component. + ## Where to start | To | Read | @@ -58,7 +60,7 @@ The shaded box is the Publisher. | Offer a platform | [Managing adapters](platforms/manage.md) | | Relay SMS to the Publisher | [Managing gateway clients](gateway-clients/manage.md) | | Build a client | [gRPC API](reference/grpc-api.md) and [REST API](reference/rest-api.md) | -| Understand the protocol | [RelaySMS whitepaper](explanation/relaysms-whitepaper.pdf) | +| Understand the protocol | [Security model](explanation/security-model.md) and the [RelaySMS whitepaper](explanation/relaysms-whitepaper.pdf) | ## Getting help diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index fb63c38..b67ba52 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -51,18 +51,18 @@ # Reference -- [Command-line tools]() +- [Command-line tools](reference/cli.md) - [REST API](reference/rest-api.md) - [gRPC API](reference/grpc-api.md) -- [Glossary]() +- [Glossary](reference/glossary.md) # Explanation -- [Architecture]() -- [Security model]() -- [Data and retention]() +- [Architecture](explanation/architecture.md) +- [Security model](explanation/security-model.md) +- [Data and retention](explanation/data-retention.md) # Developers -- [Writing an adapter]() +- [Writing an adapter](developers/writing-an-adapter.md) - [Reference client](developers/reference-client.md) diff --git a/docs/developers/writing-an-adapter.md b/docs/developers/writing-an-adapter.md new file mode 100644 index 0000000..f4f259b --- /dev/null +++ b/docs/developers/writing-an-adapter.md @@ -0,0 +1,30 @@ +# Writing an adapter + +An adapter publishes to one platform. It's a Python package in its own repository, built with the [RelaySMS Adapter SDK](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk), which the Publisher runs in its own virtualenv and calls over JSON-RPC. + +## Requirements + +- Python 3.12 or newer +- A GitHub repository +- A developer account and API credentials for the platform + +## Steps + +1. Copy the SDK's [echo adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk/examples/echo-adapter) into a new repository named after the platform and protocol, such as `example-oauth2-adapter`. +2. Subclass `OAuth2Adapter` or `PNBAAdapter` and implement its methods ([Write an adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#write-an-adapter)). +3. Describe it in `adapter.toml`: its name, protocol, category, and a `[config]` table for each setting ([Describe it](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#describe-it)). +4. Try it with the `relaysms-adapter` console, which runs it the way the Publisher does ([Try it](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#try-it)). +5. Tag a release, such as `v1.0.0` ([Release an adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#release-an-adapter)). +6. Install it on a Publisher: + + ```bash + sudo ./publisher.sh platforms add https://github.com// + ``` + +7. Set its settings with `sudo ./publisher.sh platforms config set =`, or `platforms setup ` if it has a setup ([Settings](../platforms/manage.md#settings)). + +## Check it + +`sudo ./publisher.sh platforms list` shows the adapter with its tag, and ✓ under Configured once its settings are set. Then link a test account from the RelaySMS app. + +The five adapters in the [smswithoutborders](https://github.com/smswithoutborders) organization, such as [gmail-oauth2-adapter](https://github.com/smswithoutborders/gmail-oauth2-adapter), are complete examples. diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md new file mode 100644 index 0000000..1475269 --- /dev/null +++ b/docs/explanation/architecture.md @@ -0,0 +1,67 @@ +# Architecture + +The Publisher is five services sharing one database and one Celery broker. Each platform is reached through an adapter that runs as its own process. + +```mermaid +--- +config: + flowchart: + curve: linear +--- +flowchart TB + app["RelaySMS app"] + gateway["Gateway client"] + + subgraph publisher[" "] + grpc["gRPC API"] + rest["REST API"] + smtp["Email listener"] + broker[("Celery broker")] + worker["Celery worker"] + beat["Celery Beat"] + adapter["Adapter process"] + end + + platform["Gmail, Telegram, ..."] + + app -- "link an account" --> grpc + app -- "SMS" --> gateway + gateway -- "HTTPS" --> rest + gateway -- "email" --> smtp + rest -- "queue" --> broker + smtp -- "queue" --> broker + beat -- "schedule cleanups" --> broker + broker --> worker + grpc --> adapter + worker --> adapter + adapter --> platform +``` + +The shaded box is the Publisher. Every service in it also reads and writes the database. + +## Services + +| Service | Does | +| --- | --- | +| gRPC API | Links and unlinks accounts through their adapters, stores their tokens encrypted, and exchanges ephemeral public keys with the app (`SyncKeys`) | +| REST API | Takes payloads from gateway clients, and serves the admin endpoints, publication stats and public lists | +| Email listener | Reads payloads that gateway clients email to a mailbox | +| Celery worker | Decrypts each queued payload and publishes it through the platform's adapter; also installs and updates adapters | +| Celery Beat | Schedules the cleanup jobs and, when set, the worker heartbeat | + +The database holds linked accounts, keys, credentials, adapters, gateway clients, the audit log and publication stats. The broker carries the queue between the services and the worker; with the default SQLite broker it's two files in `data/`. + +## Adapters + +Each adapter is a Python package installed from its own repository into its own virtualenv. The Publisher runs it as a separate process for each call, with a 60-second limit, and talks to it over JSON-RPC on standard input and output. It gets its own config and state directories under `data/platforms/` and a few environment variables, not the Publisher's settings, but it runs as the service user, so it can read `.env` and the database. See [Writing an adapter](../developers/writing-an-adapter.md). + +## A publication, step by step + +1. A gateway client posts a payload to `POST /v1/publications`, or emails it. +2. The REST API or the email listener checks its structure, replaces the sender's address with a keyed hash, and queues it. +3. A multi-part payload waits until all its segments arrive. +4. The worker finds the payload's keys, decrypts it, and calls the platform's adapter to publish it. +5. Only a successful publication uses up the keys; a failure leaves them usable. +6. The outcome is recorded in the publication stats. + +The [security model](security-model.md) covers the keys and encryption. diff --git a/docs/explanation/data-retention.md b/docs/explanation/data-retention.md new file mode 100644 index 0000000..c1b65b0 --- /dev/null +++ b/docs/explanation/data-retention.md @@ -0,0 +1,19 @@ +# Data and retention + +What the Publisher stores and how long it keeps it. The cleanup jobs run on the [`CELERY_CLEANUP_CRON` and `CELERY_TOKEN_CLEANUP_CRON` schedules](../configuration/settings.md#cleanup-jobs), and delete data past its limit at the next run. + +| Data | Kept until | +| --- | --- | +| Linked accounts' tokens | The user unlinks, or the account has no publication or key sync for `TOKEN_IDLE_MAX_AGE_DAYS` (90); then the Publisher also tries to revoke it on its platform | +| Ephemeral key pools | Each pair is deleted when a publication or token check uses it; the pools go with their token | +| Incomplete multi-part payloads | All segments arrive, or `PAYLOAD_SESSION_MAX_AGE_HOURS` (3) passes | +| Publication stats | Kept | +| Audit events and adapter jobs | `AUDIT_RETENTION_DAYS` (365) | +| Web sessions | Logout, `AUTH_SESSION_IDLE_MINUTES` (30) idle, or `AUTH_SESSION_MAX_HOURS` (12) | +| Credentials, adapters and gateway clients | You remove them | +| Queued payloads | With the SQLite broker, kept in `data/celery_broker.db` after delivery, with the payload ciphertext, sender hash, country code and any `tag`; Redis and RabbitMQ delete them once delivered | +| Server identity keys | Kept | +| Adapters' own files | Up to each adapter; they live in its state directory | +| Logs | The system journal's retention, and SigNoz's when `OTEL_EXPORTER_OTLP_ENDPOINT` is set | + +See the [security model](security-model.md) for what each record holds. diff --git a/docs/explanation/security-model.md b/docs/explanation/security-model.md new file mode 100644 index 0000000..c34e28c --- /dev/null +++ b/docs/explanation/security-model.md @@ -0,0 +1,38 @@ +# Security model + +The RelaySMS protocol is described in the [technical whitepaper](relaysms-whitepaper.pdf). This page covers what the Publisher holds and who can read it. + +## Keys + +| Key | Where | Used for | +| --- | --- | --- | +| Server identity keys | 256 X25519 key pairs in the database | Every gRPC request and every payload; the only key for payloads sent without a linked account | +| Ephemeral key pools | Up to 256 client public keys and 256 server key pairs per linked account | One use each, by a payload or a token check; a successful use deletes the pair | +| [Encryption keys](../configuration/encryption-keys.md) | `.env` | Encrypting the private keys, the platform tokens and, with SQLite, the database file, and keying the sender hash | + +The app refills a linked account's pools with the gRPC `SyncKeys` call. A failed publication leaves its key pair usable, so the same SMS can publish when it's sent again; the whitepaper's section 4.3 deletes keys on failure too. A client must never encrypt different content with a key pair it has already used. + +## What travels encrypted + +- **Payloads from a linked account** are encrypted by the app with a pool key pair combined with an identity key, so the gateway client and the network carry only ciphertext. +- **Payloads sent without a linked account** are encrypted to an identity key alone, without forward secrecy: anyone who later obtains the identity private keys can decrypt recorded ones. +- **gRPC requests** carry their payload (`x-payload-bin`) encrypted to an identity key. A nonce seen within the last `NONCE_TTL_SECONDS` is refused, and so is a header timestamp more than that far off; the seen nonces are kept in memory, so the list starts empty after a restart. The other request fields, such as phone numbers, login codes and two-step passwords, travel in plain protobuf, protected by TLS from nginx or `GRPC_TLS_ENABLED`. +- **The REST API** takes payloads without credentials, and **the email listener** only from [allowed senders](../configuration/smtp-transport.md#allowlist). Either way, a payload only publishes if it decrypts with a linked account's keys, or passes the [offline publishing](../configuration/offline-publishing.md) rules. + +## What the operator can see + +| Data | Stored as | +| --- | --- | +| Platform tokens | Encrypted with `DATABASE_FIELD_ENCRYPTION_KEY` while `DATABASE_FIELD_ENCRYPTION_ENABLED` is on, as `template.env` sets it | +| Private keys | Encrypted with `DATA_ENCRYPTION_KEY` | +| Sender phone numbers | Replaced with a keyed hash before anything is stored | +| Message content | Ciphertext in the queue ([kept with the SQLite broker](data-retention.md)), and while a multi-part payload waits for its segments; decrypted only in memory, in the worker and the adapter process it hands it to | +| Publication stats | Platform, protocol, status, country code, failure reason and time | + +## Adapters + +An adapter runs as the service user, so it can read `.env` and the database. Install only adapters you trust, from orgs you list in `PLATFORMS_GITHUB_ORGS`. + +## Who can read what + +Anyone holding both `.env` and the database can decrypt the tokens, publish as the linked accounts, and decrypt recorded payloads sent without a linked account. With field encryption off, the database alone exposes the tokens. Protect them as [Backup and restore](../operating/backup-restore.md) and [Security hardening](../operating/security-hardening.md) describe. diff --git a/docs/platforms/manage.md b/docs/platforms/manage.md index d264f5e..c3f0d37 100644 --- a/docs/platforms/manage.md +++ b/docs/platforms/manage.md @@ -99,4 +99,4 @@ sudo ./publisher.sh platforms exec -- --help ## Writing an adapter -Build it with the [adapter SDK](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk), then tag a release and `add` it. +See [Writing an adapter](../developers/writing-an-adapter.md). diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 0000000..780e1e7 --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,31 @@ +# Command-line tools + +| Tool | Runs | Documented in | +| --- | --- | --- | +| `install.sh` | Installs or updates a server install | [Install on a server](../installation/server.md) | +| `manage.sh` | Controls, updates and checks the services | [Services](../operating/services.md#managesh) | +| `publisher.sh` | The admin CLI, as the service user | [Below](#publishersh) | +| `sync-env.sh` | Adds to `.env` the settings `template.env` has and it lacks | [Upgrading](../operating/upgrading.md) | +| `scripts/setup-postgres.sh`, `scripts/setup-mysql.sh` | Sets up a database server | [Database](../installation/database.md) | +| `scripts/setup-rabbitmq.sh` | Sets up a RabbitMQ broker | [Celery broker](../installation/celery-broker.md) | +| `scripts/setup-observability.sh` | Sets up SigNoz and Uptime Kuma | [Observability](../operating/observability.md) | + +Run them with `sudo`; all but `install.sh` from the install directory. Each takes `--help`. + +## manage.sh + +`status`, `logs`, `start`, `stop`, `restart`, `enable`, `disable`, `update`, `migrate`, `check`, `nginx`, `docs`, `uninstall`. See [Services](../operating/services.md#managesh). + +## publisher.sh + +Runs `python3 -m publisher` as the service user. `sudo ./publisher.sh --help` describes each command. + +| Group | Commands | Documented in | +| --- | --- | --- | +| `creds` | `create`, `list`, `scopes`, `set-scopes`, `reset-password`, `disable`, `enable`, `revoke-sessions`, `delete` | [Credentials and access](../operating/credentials.md) | +| `platforms` | `add`, `update`, `list`, `disable`, `enable`, `remove`, `config show`, `config set`, `setup`, `import`, `exec` | [Managing adapters](../platforms/manage.md) | +| `gateway-clients` | `suggest`, `create`, `list`, `update`, `disable`, `enable`, `delete`, `countries`, `operators`, `mcc-mnc list` | [Managing gateway clients](../gateway-clients/manage.md) | +| `config` | `check` | [Settings reference](../configuration/settings.md) | +| `seed` | `stats`, `creds`, `platforms`, `gateway-clients` | None; fills a development database with sample data | + +It also takes `env` and `shell` ([Services](../operating/services.md#publishersh)). diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md new file mode 100644 index 0000000..1c3c0c8 --- /dev/null +++ b/docs/reference/glossary.md @@ -0,0 +1,24 @@ +# Glossary + +- **Adapter** (platform adapter): the plugin that publishes to one platform, such as Gmail, installed from its own repository. See [Managing adapters](../platforms/manage.md). +- **Administrator**: a credential holding every scope. +- **Audit event**: a record of a change or login. See [Audit log](../operating/audit-log.md). +- **Category**: what an adapter publishes: email, message, text or bridge (`--cat-id`). +- **Credential**: a username and password that logs in to the REST API's admin endpoints, with its own scopes. See [Credentials and access](../operating/credentials.md). +- **Ephemeral key pool**: the client public keys and server key pairs each linked account holds, up to 256 of each; each pair is used once, with an identity key, by a payload or a token check. See [Security model](../explanation/security-model.md). +- **Gateway client**: a phone number that relays SMS to the Publisher, from an Android phone running Deku SMS or from Twilio. See [Connecting a relay](../gateway-clients/connect-a-relay.md). +- **Identity key**: one of the server's 256 long-term X25519 keys, used in every gRPC request and payload, and alone for offline payloads. +- **Linked account**: a user's account on a platform, authorized once while online. The Publisher stores its token encrypted. +- **MSISDN**: a phone number in international format, such as `+237600000000`. +- **OAuth2**: linking through the platform's consent page, as Gmail, Mastodon and Bluesky do. +- **Offline payload**: content sent without a linked account, published through RelaySMS Mail; the whitepaper calls these offline-first publications. See [Offline publishing](../configuration/offline-publishing.md). +- **Payload**: the encrypted content the RelaySMS app sends by SMS; one carrying an attachment arrives in several segments. +- **PLMN**: a mobile network, identified by its MCC (country code) and MNC (network code). See [How suggest works](../gateway-clients/manage.md#how-suggest-works). +- **PNBA** (phone number-based authentication): linking with a code sent to the user's phone, as Telegram and RelaySMS Mail do. +- **Protocol**: how an adapter links accounts, `oauth2` or `pnba` (`--proto-id`). For a payload, how it reached the Publisher: `https` (the REST API) or `smtp` (the email listener). +- **Redirect URI**: where an OAuth2 platform sends the user back after they authorize. See [Redirect URIs](../platforms/manage.md#redirect-uris). +- **rmail**: the name of the RelaySMS Mail adapter. See [RelaySMS Mail](../platforms/relaysms-mail.md). +- **Scope**: a permission a credential holds, such as `gc:write`. See [Scopes](../operating/credentials.md#scopes). +- **Sender hash**: the keyed hash that replaces a sender's phone number before anything is stored. +- **Token**: what the Publisher stores for a linked account to act on the platform, encrypted. +- **Web session**: a REST API login kept in a cookie, for a web client. diff --git a/tests/cli/test_reference.py b/tests/cli/test_reference.py new file mode 100644 index 0000000..c712658 --- /dev/null +++ b/tests/cli/test_reference.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-3.0-only + +import re + +import click + +from publisher import config +from publisher.cli import cli + + +def reference_section(heading: str) -> str: + text = (config.ROOT / "docs" / "reference" / "cli.md").read_text() + return text.split(f"## {heading}", 1)[1].split("\n## ", 1)[0] + + +def commands(group: click.Group, prefix: str = "") -> set[str]: + names = set() + for name, command in group.commands.items(): + if isinstance(command, click.Group): + names |= commands(command, f"{prefix}{name} ") + else: + names.add(f"{prefix}{name}") + return names + + +def test_cli_reference_lists_every_publisher_command(): + section = reference_section("publisher.sh") + documented = set() + for group, cells in re.findall(r"^\| `([a-z-]+)` \| ([^|]+) \|", section, re.M): + documented |= {f"{group} {name}" for name in re.findall(r"`([a-z -]+)`", cells)} + + assert commands(cli) - documented == set(), "missing from docs/reference/cli.md" + assert documented - commands(cli) == set(), "in docs/reference/cli.md, not the CLI" + + +def test_cli_reference_lists_every_manage_command(): + usage = re.search( + r"Usage: \$0 \{([^}]+)\}", (config.ROOT / "manage.sh").read_text() + ) + assert usage + actual = set(usage[1].split("|")) + documented = set(re.findall(r"`([a-z]+)`", reference_section("manage.sh"))) + + assert actual - documented == set(), "missing from docs/reference/cli.md" + assert documented - actual == set(), "in docs/reference/cli.md, not manage.sh" From db3383934a8a9ddb214b37f630434d0fd934d7d7 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Fri, 23 Oct 2026 16:12:48 +0100 Subject: [PATCH 13/13] docs: add consistent next steps, drop internal notes and closed lists, tighten prose --- CONTRIBUTING.md | 7 +- README.md | 200 +++-------------------- docs/README.md | 26 ++- docs/configuration/encryption-keys.md | 2 - docs/configuration/offline-publishing.md | 8 +- docs/configuration/settings.md | 6 +- docs/configuration/smtp-transport.md | 8 +- docs/developers/reference-client.md | 16 +- docs/developers/writing-an-adapter.md | 4 +- docs/explanation/architecture.md | 22 ++- docs/explanation/data-retention.md | 3 +- docs/explanation/security-model.md | 8 +- docs/gateway-clients/connect-a-relay.md | 9 +- docs/gateway-clients/manage.md | 8 +- docs/installation/celery-broker.md | 2 +- docs/installation/database.md | 6 +- docs/installation/docker.md | 10 +- docs/installation/manual.md | 4 +- docs/installation/multiple-instances.md | 4 +- docs/installation/requirements.md | 4 +- docs/installation/reverse-proxy.md | 6 +- docs/installation/server.md | 52 +++--- docs/operating/backup-restore.md | 6 +- docs/operating/credentials.md | 2 +- docs/operating/observability.md | 4 +- docs/operating/security-hardening.md | 2 +- docs/operating/services.md | 14 +- docs/operating/troubleshooting.md | 23 ++- docs/operating/upgrading.md | 26 +-- docs/platforms/manage.md | 15 +- docs/platforms/relaysms-mail.md | 2 +- docs/platforms/telegram.md | 2 +- docs/reference/cli.md | 2 +- docs/reference/glossary.md | 12 +- docs/reference/grpc-api.md | 139 ++++++++-------- docs/reference/rest-api.md | 12 +- publisher.sh | 2 +- template.env | 14 +- tests/cli/test_reference.py | 15 +- 39 files changed, 294 insertions(+), 413 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6f583ee..9183792 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -176,7 +176,7 @@ Common types are `feat`, `fix`, `refactor`, `test`, `docs`, `build`, `ci` and `c - [ ] The PR targets `staging` and does one thing. - [ ] `make check` and `make test` pass locally. - [ ] New behavior and bug fixes come with tests. A bug fix has a test that fails without it. -- [ ] Docs, `template.env` and the README are updated if behavior, settings or commands changed. +- [ ] The manual in `docs/` and `template.env` are updated if behavior, settings or commands changed. - [ ] Schema changes include a migration, and a commit message with `[migrate]`. - [ ] The description says what changed, why, and how it was tested. Link the issue with `Closes #123`. - [ ] Anything operators must do on deploy is called out. @@ -293,7 +293,7 @@ make build test PYTHON=.venv312/bin/python ### Add a setting 1. Add the field to its section in `publisher/config.py` and read it in that section's `load`. -2. Add it, commented, to `template.env` and to the Configuration section of the README. +2. Add it, commented, to `template.env` and to [docs/configuration/settings.md](docs/configuration/settings.md); a test fails until both list it. 3. Check validation with `python -m publisher config check`, and add a test in `tests/test_config.py`. ### Change the database schema @@ -340,7 +340,8 @@ The manual in `docs/` is an [mdBook](https://rust-lang.github.io/mdBook/) that r - List a new page in `docs/SUMMARY.md`, or the book leaves it out. - Link pages in `docs/` relatively, and files outside it by their full GitHub URL, since the book only has `docs/`. - Draw diagrams as `mermaid` code blocks. -- Start a task page with its requirements, and end it with a link to the next step when that isn't the following page. +- Say "web client" and "public", and use `publisher.example.com`, `example.com` and `+237600000000` in examples. Keep GitHub alerts for what readers must not miss. +- Start a task page with its requirements. End a page that points onward with a `## Next steps` section of one or two linked sentences or a short list; skip it when the book's next page is the only next step. - Use tables for reference data, numbered steps for procedures, and one heading per symptom for troubleshooting. - Write plainly and briefly in US English, one paragraph per line, with sentence-case headings and `` placeholders. diff --git a/README.md b/README.md index 08a911f..de2e2fc 100644 --- a/README.md +++ b/README.md @@ -1,148 +1,33 @@ # RelaySMS Publisher -Publishes to online platforms (Gmail, X, Telegram and more) from content sent by SMS, so people can post without an internet connection. +Publishes to online platforms, such as Gmail, from content people send by SMS, so they can post without an internet connection. See [how it works](docs/README.md#how-it-works). -## Contents +## Install -- [How it works](#how-it-works) -- [Requirements](#requirements) -- [Install on a server](#install-on-a-server) -- [Manage the services](#manage-the-services) -- [Run with Docker](#run-with-docker) -- [Development](#development) -- [Configuration](#configuration) -- [Platform adapters](#platform-adapters) -- [Gateway clients](#gateway-clients) -- [Credentials](#credentials) -- [Documentation](#documentation) -- [License](#license) +On Ubuntu 24.04 or Debian 13: -## How it works - -1. A user links an account (Gmail, X, ...) through the gRPC API, which stores its token encrypted. -2. Offline, the user's app encrypts the content and sends it by SMS to a gateway client: a phone number that relays SMS to this server. -3. The payload arrives over REST or SMTP and is queued. -4. A Celery worker decrypts it and publishes it through the platform's adapter. - -## Requirements - -- Ubuntu 24.04 or Debian 13 -- Python 3.12 or newer -- A database: SQLite (the default), MySQL 8.0.28 or newer, MariaDB, or PostgreSQL 12 or newer - -The installer sets up everything else, including the system packages and Rust. - -## Install on a server - -1. Run the installer. It installs to `/opt/relaysms/relaysms-publisher`, generates the encryption keys and starts the services under systemd: - - ```bash - curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | sudo bash - ``` - - It asks about each option you don't pass as a flag. To choose up front, pass flags after `bash -s --`: - - ```bash - curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | \ - sudo bash -s -- --site-name publisher.example.com --setup-db postgres - ``` - - | Flag | Does | - | --- | --- | - | `--site-name DOMAIN` | Puts nginx with a Let's Encrypt certificate in front; `--skip-nginx` skips it | - | `--setup-db mysql\|postgres` | Installs the database server and writes its details to `.env` ([details](docs/installation/database.md)) | - | `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker, for heavier load ([details](docs/installation/celery-broker.md)) | - | `--docs-site-name DOMAIN` | Serves the manual in `docs/` on its own domain, rebuilt by every `manage.sh update` | - | `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](docs/operating/observability.md)) | - | `--install-dir PATH`, `--instance-name NAME` | Installs elsewhere, or a second copy on the same host ([details](docs/installation/multiple-instances.md)) | - - `install.sh --help` lists every flag. - -2. Go to the install directory and check the configuration: - - ```bash - cd /opt/relaysms/relaysms-publisher - sudo ./manage.sh check - ``` - -3. Create an administrator credential for the REST API: - - ```bash - sudo ./publisher.sh creds create --username ops --administrator - ``` - - > [!IMPORTANT] - > The password is printed only once. - -4. Add the platform adapters you want to offer, then configure each with its [provider guide](docs/platforms/manage.md#provider-guides): - - ```bash - sudo ./publisher.sh platforms add https://github.com/smswithoutborders/gmail-oauth2-adapter - ``` - -5. Register the gateway clients that relay SMS to this server: - - ```bash - sudo ./publisher.sh gateway-clients create --msisdn +237123456789 --protocols https - ``` - -6. Check that the services are up: - - ```bash - sudo ./manage.sh status - curl http://127.0.0.1:16000/health - ``` - -[Installation](docs/installation/server.md) covers the installer, installing by hand, databases and nginx. - -## Manage the services - -From the install directory: - -| Command | Does | -| --- | --- | -| `sudo ./manage.sh status` | Show each service's state | -| `sudo ./manage.sh logs` | Follow the service logs | -| `sudo ./manage.sh start`, `stop`, `restart` | Control every service together | -| `sudo ./manage.sh enable`, `disable` | Start the services on boot, or stop doing so | -| `sudo ./manage.sh update` | Pull the latest code, rebuild and restart; add `--migrate` when the release has migrations | -| `sudo ./manage.sh migrate` | Apply database migrations | -| `sudo ./manage.sh check` | Report every missing or invalid setting | -| `sudo ./manage.sh nginx [DOMAIN]` | Re-render the nginx site and reattach or obtain its certificate | -| `sudo ./manage.sh docs [DOMAIN]` | Build the manual and serve it on its own domain | -| `sudo ./manage.sh uninstall` | Remove the services and the install directory | - -See [Services](docs/operating/services.md) for log filters and what `update` and `nginx` do. - -## Run with Docker - -1. Clone the repository with its submodule: - - ```bash - git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git - cd RelaySMS-Publisher - ``` - -2. Create `.env` with fresh encryption keys: +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash +``` - ```bash - make .env - ``` +[Install on a server](docs/installation/server.md) covers the installer's options and the steps after it. To try it on your machine with Docker first, follow the [quick start](docs/getting-started/quick-start.md). -3. Build and start: +## Documentation - ```bash - docker compose up -d --build - ``` +The [manual](docs/README.md) starts with: -The container applies pending migrations, then starts every service. It listens on `PORT` (16000) and `GRPC_PORT` (6000) from `.env`, and keeps its data in `./data`. +- [Quick start](docs/getting-started/quick-start.md) +- [Settings reference](docs/configuration/settings.md) +- [Managing adapters](docs/platforms/manage.md) and the [provider guides](docs/platforms/manage.md#provider-guides) +- [Connecting a relay](docs/gateway-clients/connect-a-relay.md) +- [Troubleshooting](docs/operating/troubleshooting.md) -> [!CAUTION] -> Back up `.env` with `./data`: its [encryption keys](docs/configuration/encryption-keys.md) are the only way to read the data. +`sudo ./manage.sh docs ` serves the manual from your own server. -## Development +## Contributing -[CONTRIBUTING.md](CONTRIBUTING.md) walks through setting up, running the services locally, testing and making common changes. In short: +[CONTRIBUTING.md](CONTRIBUTING.md) covers setup, running the services locally, testing and the workflow. To start: ```bash git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git @@ -150,53 +35,12 @@ cd RelaySMS-Publisher make setup ``` -Then follow [Run locally](CONTRIBUTING.md#run-locally). To fill a local database with sample data: - -```bash -source venv/bin/activate -python -m publisher seed stats --count 5000 --days 90 # publication stats -python -m publisher seed creds --count 5 # credentials; prints their passwords -python -m publisher seed platforms # adapter rows, without their files -python -m publisher seed gateway-clients --count 10 # gateway clients, some disabled -``` - -## Configuration - -Settings live in `.env`. The [settings reference](docs/configuration/settings.md) lists each one, and `sudo ./manage.sh check` reports any that are missing or invalid. - -## Platform adapters - -Each platform is served by an adapter installed from its own repository. `GET /v1/platforms` lists the enabled ones. - -- [Gmail](https://github.com/smswithoutborders/gmail-oauth2-adapter) -- [X (formerly Twitter)](https://github.com/smswithoutborders/twitter-oauth2-adapter) -- [Telegram](https://github.com/smswithoutborders/telegram-pnba-adapter) -- [Slack](https://github.com/smswithoutborders/slack-oauth2-adapter) -- [Bluesky](https://github.com/smswithoutborders/bluesky-oauth2-adapter) -- [Mastodon](https://github.com/smswithoutborders/mastodon-oauth2-adapter) +Adapters are built with the [adapter SDK](sdk/README.md). -[Managing adapters](docs/platforms/manage.md) covers adding, updating, disabling and removing adapters, and links a guide for each. - -## Gateway clients - -Gateway clients are the phone numbers that relay SMS to this server. `GET /v1/gateway-clients` lists the enabled ones, and [docs/gateway-clients/manage.md](docs/gateway-clients/manage.md) covers managing them. - -## Credentials - -Credentials log in to the REST API's admin endpoints, each with its own scopes. See [Credentials and access](docs/operating/credentials.md). - -## Documentation +## Security -- [Manual](docs/README.md): installing, configuring and running the Publisher -- [REST API](docs/reference/rest-api.md): publishing, authentication and the [OpenAPI reference](docs/openapi.json) -- [gRPC API](docs/reference/grpc-api.md): linking accounts and syncing keys -- [SMTP transport](docs/configuration/smtp-transport.md): publishing payloads sent by email -- [Platform adapters](docs/platforms/manage.md): installing and managing adapters -- [Gateway clients](docs/gateway-clients/manage.md): managing gateway clients -- [Observability](docs/operating/observability.md): tracing, metrics, logs and uptime monitoring -- [Reference client](docs/developers/reference-client.md): exercising the gRPC and REST flows by hand -- [Contributing](CONTRIBUTING.md): setup, workflow, conventions and testing +Report vulnerabilities privately, as [SECURITY.md](SECURITY.md) describes. ## License -Licensed under the GNU General Public License v3. See [LICENSE](LICENSE.md). +Licensed under the GNU General Public License v3.0. See [LICENSE.md](LICENSE.md). diff --git a/docs/README.md b/docs/README.md index 75335ba..39c4401 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # RelaySMS Publisher -The Publisher posts to online platforms (Gmail, Telegram, Mastodon and more) from content people send by SMS, so they can publish without an internet connection. +Publishes to online platforms, such as Gmail, from content people send by SMS, so they can post without an internet connection. ## How it works @@ -13,28 +13,36 @@ config: flowchart TB app["RelaySMS app"] gateway["Gateway client"] - platform["Gmail, Telegram, ..."] + mailbox["Mailbox"] + platform["Online platforms"] - subgraph publisher[" "] + subgraph publisher["Publisher"] grpc["gRPC API"] rest["REST API"] smtp["Email listener"] + db[("Database")] worker["Celery worker"] adapter["Platform adapter"] + %% Keeps the title clear of the arrows entering the box. + spacer[" "]:::hidden + spacer ~~~ worker end app -- "link an account, online" --> grpc app -- "encrypted SMS" --> gateway gateway -- "HTTPS" --> rest - gateway -- "email, via a mailbox" --> smtp + gateway -- "email" --> mailbox + mailbox -- "IMAP" --> smtp + grpc -- "token, keys" --> db + db --> worker rest --> worker smtp --> worker grpc --> adapter worker --> adapter adapter --> platform -``` -The shaded box is the Publisher. + classDef hidden fill:none,stroke:none +``` 1. While online, a user links an account through the gRPC API. The Publisher stores its token encrypted. 2. Offline, the app encrypts the content and sends it by SMS to a gateway client, a phone number that relays SMS. @@ -43,11 +51,11 @@ The shaded box is the Publisher. ## Key terms -- **Gateway client**: a phone number that relays SMS to the Publisher, from an Android phone running Deku SMS or from Twilio. See [Connecting a relay](gateway-clients/connect-a-relay.md). +- **Gateway client**: a phone number that relays SMS to the Publisher, such as an Android phone running Deku SMS. See [Connecting a relay](gateway-clients/connect-a-relay.md). - **Adapter** (platform adapter): the plugin that publishes to one platform, such as Gmail, installed from its own repository. See [Managing adapters](platforms/manage.md). - **Linked account**: a user's account on a platform, authorized once while online. The Publisher stores its token encrypted. -- **OAuth2 and PNBA**: the two ways an account links. OAuth2 goes through the platform's consent page (Gmail, Mastodon, Bluesky). Phone number-based authentication (PNBA) uses a code sent to the user's phone (Telegram, RelaySMS Mail). -- **Offline payload**: content sent without a linked account, published through [RelaySMS Mail](platforms/relaysms-mail.md). +- **OAuth2 and PNBA**: how accounts link. OAuth2 goes through the platform's consent page, as for Gmail. Phone number-based authentication (PNBA) uses a code sent to the user's phone, as for Telegram. The [provider guides](platforms/manage.md#provider-guides) give each platform's protocol. +- **Offline payload**: content sent without a linked account, published through [RelaySMS Mail](platforms/relaysms-mail.md). See [Offline publishing](configuration/offline-publishing.md). The [glossary](reference/glossary.md) defines the rest, and [Architecture](explanation/architecture.md) shows every component. diff --git a/docs/configuration/encryption-keys.md b/docs/configuration/encryption-keys.md index c80eaf2..cf99b82 100644 --- a/docs/configuration/encryption-keys.md +++ b/docs/configuration/encryption-keys.md @@ -8,7 +8,5 @@ Three keys in `.env` protect the data. The installer and `make .env` generate th | `DATABASE_FIELD_ENCRYPTION_KEY` | Linked accounts' platform tokens | When field encryption is on | | `DATABASE_ENCRYPTION_KEY` | The whole SQLite database file (SQLCipher) | When encryption is on and the database is SQLite | -`template.env` turns both on with `DATABASE_FIELD_ENCRYPTION_ENABLED=true` and `DATABASE_ENCRYPTION_ENABLED=true`. - > [!CAUTION] > Back up `.env` whenever you back up the database, store it apart ([Backup and restore](../operating/backup-restore.md)), and never change a key once the install holds data. Data encrypted with a lost or changed key can't be read again, and a new `DATA_ENCRYPTION_KEY` also gives every sender a new hash. diff --git a/docs/configuration/offline-publishing.md b/docs/configuration/offline-publishing.md index 79e131d..cf23460 100644 --- a/docs/configuration/offline-publishing.md +++ b/docs/configuration/offline-publishing.md @@ -1,14 +1,12 @@ # Offline publishing -Use this to let people send email through [RelaySMS Mail](../platforms/relaysms-mail.md) without linking an account first. Payloads from linked accounts, such as Gmail ones, need none of these settings. An offline payload carries no linked account: the app encrypts it to the server's identity key, and the Publisher publishes it through the RelaySMS Mail adapter (`rmail`), which must be installed. Anyone who can reach an allowed protocol can send one. +Offline payloads let people send email through [RelaySMS Mail](../platforms/relaysms-mail.md) without linking an account; linked accounts need none of this ([settings](settings.md#offline-publishing)). The app encrypts an offline payload to the server's identity key, and the Publisher publishes it through the `rmail` adapter, which must be installed. Anyone who can reach an allowed protocol can send one. Each payload is tagged with the protocol it arrived on: - `https`: [REST `POST /v1/publications`](../reference/rest-api.md#publishing) - `smtp`: the [SMTP transport](smtp-transport.md) -Settings: [Offline publishing](settings.md#offline-publishing). - - An offline payload from a protocol not in `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` is discarded. - An offline payload over `https` must carry `OFFLINE_PUBLISH_SHARED_SECRET` in the request's `tag` field, or it's discarded. - Leaving `OFFLINE_PUBLISH_ALLOWED_PROTOCOLS` empty, or listing `https`, without the secret fails `manage.sh check`, and the services refuse to start. @@ -30,6 +28,6 @@ Settings: [Offline publishing](settings.md#offline-publishing). sudo ./manage.sh restart ``` -4. Give the secret to each gateway client that posts to `POST /v1/publications`, to send as `tag`. +## Next steps -For a Twilio number, [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) can post to this server: set its `FORWARD_URLS_JSON` to `https:///v1/publications` and `FORWARD_TAG` to the secret. +[Connect each relay](../gateway-clients/connect-a-relay.md) that posts over `https` with the secret as its `tag` (`FORWARD_TAG` for Twilio). diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 4b2527b..30a1693 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -13,7 +13,7 @@ Settings live in `.env` in the install directory. `template.env` lists each one | --- | --- | --- | | `HOST` | `127.0.0.1` | REST API address. Docker always binds `0.0.0.0`. | | `PORT` | `16000` | REST API port | -| `WORKERS` | None | REST API worker processes: `1` by default under `scripts/run.sh`, `4` on Docker. The server install always runs `4`. | +| `WORKERS` | None | REST API worker processes: `1` by default under `scripts/run.sh`, `4` on Docker and server installs | | `GRPC_HOST` | `127.0.0.1` | gRPC address. Docker always binds `0.0.0.0`. | | `GRPC_PORT` | `6000` | gRPC port | | `GRPC_MAX_WORKERS` | `10` | gRPC worker threads | @@ -109,9 +109,7 @@ See [Managing adapters](../platforms/manage.md#files). Relative paths are from t | `PLATFORMS_ADAPTERS_VENV_DIR` | `data/platforms/venvs` | Directory for adapters' virtualenvs | | `PLATFORMS_ADAPTERS_CONFIG_DIR` | `data/platforms/config` | Directory for adapters' `credentials.json` files | | `PLATFORMS_ADAPTERS_STATE_DIR` | `data/platforms/state` | Directory for files adapters write, such as databases | -| `PLATFORMS_GITHUB_ORGS` | None | Comma-separated GitHub orgs that administrators may install and update adapters from over the REST API | - -With `PLATFORMS_GITHUB_ORGS` empty, the REST API refuses to install or update adapters; `publisher.sh platforms` is unaffected. +| `PLATFORMS_GITHUB_ORGS` | None | Comma-separated GitHub orgs that administrators may install and update adapters from over the REST API. Empty refuses REST installs; `publisher.sh platforms` is unaffected. | ## REST API access diff --git a/docs/configuration/smtp-transport.md b/docs/configuration/smtp-transport.md index 9a95c52..4bbf4b2 100644 --- a/docs/configuration/smtp-transport.md +++ b/docs/configuration/smtp-transport.md @@ -33,8 +33,6 @@ You should see `Connected to mailbox `. 4. Queues the payload for publication, tagged with protocol `smtp`. Whether offline payloads from email publish depends on [offline publishing](offline-publishing.md). 5. Deletes each handled email. An email that hits an unexpected error stays in the folder and is retried on the next cycle. -The `relaysms-publisher-smtp` service exits at once while `SMTP_TRANSPORT_ENABLED` is off. - ### Allowlist `SMTP_ALLOWED_SENDERS` lists exact addresses and domains, comma-separated. Empty allows nothing. @@ -44,7 +42,7 @@ The `relaysms-publisher-smtp` service exits at once while `SMTP_TRANSPORT_ENABLE ### Authentication -The listener can't re-check SPF, which needs the sender's IP. It reads the `Authentication-Results` header ([RFC 8601](https://www.rfc-editor.org/rfc/rfc8601)) that the mailbox's own server added, and only from the `authserv-id` in `SMTP_TRUSTED_AUTHSERV_ID`, since a sender can forge such a header from anyone else. `SMTP_REQUIRE_DKIM` and `SMTP_REQUIRE_SPF` choose the verdicts required. `SMTP_VERIFY_DKIM_INDEPENDENTLY=true` also re-verifies the DKIM signature against DNS. +The listener can't re-check SPF, which needs the sender's IP. It reads the `Authentication-Results` header ([RFC 8601](https://www.rfc-editor.org/rfc/rfc8601)) that the mailbox's own server added, and only from the `authserv-id` in `SMTP_TRUSTED_AUTHSERV_ID`, since a sender can forge such a header from anyone else. ## Message format @@ -64,4 +62,6 @@ The body of each email must be a JSON object: | `dialing_code` | string | No | Sender's country calling code, for publication stats | | `text` | string | Yes | Base64-encoded serialized payload | -Next: [Offline publishing](offline-publishing.md). +## Next steps + +To publish offline payloads that arrive by email, allow `smtp` in [Offline publishing](offline-publishing.md). diff --git a/docs/developers/reference-client.md b/docs/developers/reference-client.md index f2f66a4..cb44553 100644 --- a/docs/developers/reference-client.md +++ b/docs/developers/reference-client.md @@ -1,4 +1,4 @@ -# Reference Client +# Reference client `tools/client.py` drives the gRPC and REST flows by hand, the way a phone app would. It keeps tokens and key pools in `tools/db.json` between runs. @@ -14,7 +14,7 @@ python -m tools.client [ARGS] | `--host`, `--port` | `127.0.0.1`, `6000` | gRPC server | | `--tls` | off | Use TLS for gRPC | | `--rest-api` | `http://localhost:16000` | REST API base URL | -| `--platform`, `-p` | None | Platform name, such as `gmail` or `telegram` | +| `--platform`, `-p` | None | Platform name, such as `gmail` | | `--phone-number` | None | Phone number, for phone number-based authentication (PNBA), such as for Telegram | | `--request-identifier` | None | Optional request identifier | @@ -33,24 +33,24 @@ python -m tools.client [ARGS] ```bash python -m tools.client get-oauth2-url --platform gmail python -m tools.client exchange-oauth2-code --platform gmail --code -python -m tools.client get-pnba-code --platform telegram --phone-number +237123456789 -python -m tools.client exchange-pnba-code --platform telegram --phone-number +237123456789 --code +python -m tools.client get-pnba-code --platform telegram --phone-number +237600000000 +python -m tools.client exchange-pnba-code --platform telegram --phone-number +237600000000 --code ``` With several stored tokens, `sync-keys` and the revoke commands ask which one to use. Pass `--token` to choose up front. ## Publishing -`send` encrypts a message and posts it to `POST /v1/publications` with `--address` as the sender. Without an attachment it's one request; with one, the payload is split into SMS-sized segments sent `--interval` seconds apart. +`send` encrypts a message and posts it to `POST /v1/publications` with `--address` as the sender. With an attachment, the payload is split into SMS-sized segments sent `--interval` seconds apart. ```bash -python -m tools.client send --platform gmail --address +237123456789 \ +python -m tools.client send --platform gmail --address +237600000000 \ --to friend@example.com --subject "Hello" --body "Test message" -python -m tools.client send --platform gmail --address +237123456789 \ +python -m tools.client send --platform gmail --address +237600000000 \ --to friend@example.com --subject "Hello" --body "See attached" --attachment ./file.pdf --interval 2.5 --shuffle -python -m tools.client send --offline --platform rmail --address +237123456789 \ +python -m tools.client send --offline --platform rmail --address +237600000000 \ --to friend@example.com --subject "Hello" --body "No token needed" --tag ``` diff --git a/docs/developers/writing-an-adapter.md b/docs/developers/writing-an-adapter.md index f4f259b..c0d44f4 100644 --- a/docs/developers/writing-an-adapter.md +++ b/docs/developers/writing-an-adapter.md @@ -11,7 +11,7 @@ An adapter publishes to one platform. It's a Python package in its own repositor ## Steps 1. Copy the SDK's [echo adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk/examples/echo-adapter) into a new repository named after the platform and protocol, such as `example-oauth2-adapter`. -2. Subclass `OAuth2Adapter` or `PNBAAdapter` and implement its methods ([Write an adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#write-an-adapter)). +2. Subclass the SDK's base class for its protocol, such as `OAuth2Adapter`, and implement its methods ([Write an adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#write-an-adapter)). 3. Describe it in `adapter.toml`: its name, protocol, category, and a `[config]` table for each setting ([Describe it](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#describe-it)). 4. Try it with the `relaysms-adapter` console, which runs it the way the Publisher does ([Try it](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#try-it)). 5. Tag a release, such as `v1.0.0` ([Release an adapter](https://github.com/smswithoutborders/RelaySMS-Publisher/tree/main/sdk#release-an-adapter)). @@ -27,4 +27,4 @@ An adapter publishes to one platform. It's a Python package in its own repositor `sudo ./publisher.sh platforms list` shows the adapter with its tag, and ✓ under Configured once its settings are set. Then link a test account from the RelaySMS app. -The five adapters in the [smswithoutborders](https://github.com/smswithoutborders) organization, such as [gmail-oauth2-adapter](https://github.com/smswithoutborders/gmail-oauth2-adapter), are complete examples. +The adapters in the [provider guides](../platforms/manage.md#provider-guides), such as [gmail-oauth2-adapter](https://github.com/smswithoutborders/gmail-oauth2-adapter), are complete examples. diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index 1475269..67d8ffa 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -1,6 +1,6 @@ # Architecture -The Publisher is five services sharing one database and one Celery broker. Each platform is reached through an adapter that runs as its own process. +The Publisher is a set of services sharing one database and one Celery broker. Each platform is reached through an adapter that runs as its own process. ```mermaid --- @@ -11,33 +11,37 @@ config: flowchart TB app["RelaySMS app"] gateway["Gateway client"] + mailbox["Mailbox"] + platform["Online platforms"] - subgraph publisher[" "] + subgraph publisher["Publisher"] grpc["gRPC API"] rest["REST API"] smtp["Email listener"] + beat["Celery Beat"] + db[("Database")] broker[("Celery broker")] worker["Celery worker"] - beat["Celery Beat"] adapter["Adapter process"] end - platform["Gmail, Telegram, ..."] - app -- "link an account" --> grpc + grpc --> adapter app -- "SMS" --> gateway gateway -- "HTTPS" --> rest - gateway -- "email" --> smtp + gateway -- "email" --> mailbox + mailbox -- "IMAP" --> smtp rest -- "queue" --> broker smtp -- "queue" --> broker beat -- "schedule cleanups" --> broker broker --> worker - grpc --> adapter + grpc -- "token, keys" --> db + db --> worker worker --> adapter adapter --> platform ``` -The shaded box is the Publisher. Every service in it also reads and writes the database. +Every service in the Publisher also reads and writes the database. ## Services @@ -53,7 +57,7 @@ The database holds linked accounts, keys, credentials, adapters, gateway clients ## Adapters -Each adapter is a Python package installed from its own repository into its own virtualenv. The Publisher runs it as a separate process for each call, with a 60-second limit, and talks to it over JSON-RPC on standard input and output. It gets its own config and state directories under `data/platforms/` and a few environment variables, not the Publisher's settings, but it runs as the service user, so it can read `.env` and the database. See [Writing an adapter](../developers/writing-an-adapter.md). +Each adapter is a Python package installed from its own repository into its own virtualenv. The Publisher runs it as a separate process for each call, with a 60-second limit, and talks to it over JSON-RPC on standard input and output. It gets its own config and state directories under `data/platforms/` and a few environment variables, not the Publisher's settings. It runs as the service user ([security model](security-model.md#adapters)). See [Writing an adapter](../developers/writing-an-adapter.md). ## A publication, step by step diff --git a/docs/explanation/data-retention.md b/docs/explanation/data-retention.md index c1b65b0..8725d91 100644 --- a/docs/explanation/data-retention.md +++ b/docs/explanation/data-retention.md @@ -1,6 +1,6 @@ # Data and retention -What the Publisher stores and how long it keeps it. The cleanup jobs run on the [`CELERY_CLEANUP_CRON` and `CELERY_TOKEN_CLEANUP_CRON` schedules](../configuration/settings.md#cleanup-jobs), and delete data past its limit at the next run. +The cleanup jobs run on the [`CELERY_CLEANUP_CRON` and `CELERY_TOKEN_CLEANUP_CRON` schedules](../configuration/settings.md#cleanup-jobs), and delete data past its limit at the next run. | Data | Kept until | | --- | --- | @@ -11,7 +11,6 @@ What the Publisher stores and how long it keeps it. The cleanup jobs run on the | Audit events and adapter jobs | `AUDIT_RETENTION_DAYS` (365) | | Web sessions | Logout, `AUTH_SESSION_IDLE_MINUTES` (30) idle, or `AUTH_SESSION_MAX_HOURS` (12) | | Credentials, adapters and gateway clients | You remove them | -| Queued payloads | With the SQLite broker, kept in `data/celery_broker.db` after delivery, with the payload ciphertext, sender hash, country code and any `tag`; Redis and RabbitMQ delete them once delivered | | Server identity keys | Kept | | Adapters' own files | Up to each adapter; they live in its state directory | | Logs | The system journal's retention, and SigNoz's when `OTEL_EXPORTER_OTLP_ENDPOINT` is set | diff --git a/docs/explanation/security-model.md b/docs/explanation/security-model.md index c34e28c..586ec31 100644 --- a/docs/explanation/security-model.md +++ b/docs/explanation/security-model.md @@ -10,13 +10,13 @@ The RelaySMS protocol is described in the [technical whitepaper](relaysms-whitep | Ephemeral key pools | Up to 256 client public keys and 256 server key pairs per linked account | One use each, by a payload or a token check; a successful use deletes the pair | | [Encryption keys](../configuration/encryption-keys.md) | `.env` | Encrypting the private keys, the platform tokens and, with SQLite, the database file, and keying the sender hash | -The app refills a linked account's pools with the gRPC `SyncKeys` call. A failed publication leaves its key pair usable, so the same SMS can publish when it's sent again; the whitepaper's section 4.3 deletes keys on failure too. A client must never encrypt different content with a key pair it has already used. +The app refills a linked account's pools with the gRPC `SyncKeys` call. A failed publication leaves its key pair usable, so the same SMS can publish when it's sent again. A client must never encrypt different content with a key pair it has already used. ## What travels encrypted - **Payloads from a linked account** are encrypted by the app with a pool key pair combined with an identity key, so the gateway client and the network carry only ciphertext. - **Payloads sent without a linked account** are encrypted to an identity key alone, without forward secrecy: anyone who later obtains the identity private keys can decrypt recorded ones. -- **gRPC requests** carry their payload (`x-payload-bin`) encrypted to an identity key. A nonce seen within the last `NONCE_TTL_SECONDS` is refused, and so is a header timestamp more than that far off; the seen nonces are kept in memory, so the list starts empty after a restart. The other request fields, such as phone numbers, login codes and two-step passwords, travel in plain protobuf, protected by TLS from nginx or `GRPC_TLS_ENABLED`. +- **gRPC requests** carry their payload (`x-payload-bin`) encrypted to an identity key. A request is refused if its nonce was already seen, or if its timestamp is more than `NONCE_TTL_SECONDS` from the server's clock. The other request fields, such as phone numbers, login codes and two-step passwords, rely on TLS, from nginx or `GRPC_TLS_ENABLED`. - **The REST API** takes payloads without credentials, and **the email listener** only from [allowed senders](../configuration/smtp-transport.md#allowlist). Either way, a payload only publishes if it decrypts with a linked account's keys, or passes the [offline publishing](../configuration/offline-publishing.md) rules. ## What the operator can see @@ -26,12 +26,12 @@ The app refills a linked account's pools with the gRPC `SyncKeys` call. A failed | Platform tokens | Encrypted with `DATABASE_FIELD_ENCRYPTION_KEY` while `DATABASE_FIELD_ENCRYPTION_ENABLED` is on, as `template.env` sets it | | Private keys | Encrypted with `DATA_ENCRYPTION_KEY` | | Sender phone numbers | Replaced with a keyed hash before anything is stored | -| Message content | Ciphertext in the queue ([kept with the SQLite broker](data-retention.md)), and while a multi-part payload waits for its segments; decrypted only in memory, in the worker and the adapter process it hands it to | +| Message content | Ciphertext while queued, and while a multi-part payload waits for its segments; decrypted only in memory, in the worker and the adapter process it hands it to | | Publication stats | Platform, protocol, status, country code, failure reason and time | ## Adapters -An adapter runs as the service user, so it can read `.env` and the database. Install only adapters you trust, from orgs you list in `PLATFORMS_GITHUB_ORGS`. +An adapter runs as the service user, so it can read `.env` and the database. `PLATFORMS_GITHUB_ORGS` limits which orgs administrators can install from over the REST API. ## Who can read what diff --git a/docs/gateway-clients/connect-a-relay.md b/docs/gateway-clients/connect-a-relay.md index 5590e16..2593db9 100644 --- a/docs/gateway-clients/connect-a-relay.md +++ b/docs/gateway-clients/connect-a-relay.md @@ -1,6 +1,6 @@ # Connecting a relay -A gateway client receives the SMS that people send and relays each payload to the Publisher's `POST /v1/publications`. There are two: +A gateway client receives the SMS that people send and relays each payload to the Publisher's `POST /v1/publications`. Pick one: | Gateway client | Runs on | Use it for | | --- | --- | --- | @@ -20,7 +20,7 @@ A gateway client receives the SMS that people send and relays each payload to th ## Twilio -1. Install [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) as its README describes. +1. Install [RelaySMS-GatewayClient-Twilio](https://github.com/smswithoutborders/RelaySMS-GatewayClient-Twilio) as its README describes. It can run on the Publisher's server, on its own subdomain such as `twilio.example.com`. Set `TWILIO_AUTH_TOKEN` from the Twilio console, and keep a copy of the generated `SENDER_HASH_KEY`: a new key gives every sender a new address. 2. In its `.env`, set `FORWARD_URLS_JSON=https:///v1/publications`. 3. If the Publisher allows [offline publishing](../configuration/offline-publishing.md) over `https`, set `FORWARD_TAG` to its `OFFLINE_PUBLISH_SHARED_SECRET`. 4. In the Twilio console, under the number's Messaging > A message comes in, set `https:///twilio-sms` with `HTTP POST`. @@ -34,3 +34,8 @@ sudo ./manage.sh logs --unit worker ``` You should see `Published via ''.` To test without a phone, the [reference client](../developers/reference-client.md#publishing) posts a payload the way a gateway client does. + +## Next steps + +- [Back up](../operating/backup-restore.md) `.env` and `data/`. +- [Harden the install](../operating/security-hardening.md) before announcing the number. diff --git a/docs/gateway-clients/manage.md b/docs/gateway-clients/manage.md index 3d2ae8a..8e53fd1 100644 --- a/docs/gateway-clients/manage.md +++ b/docs/gateway-clients/manage.md @@ -1,6 +1,6 @@ # Managing gateway clients -A gateway client is a phone number that relays the SMS people send to this server: an Android phone running Deku SMS, or a Twilio number. Register each number here so the RelaySMS app lists it, then [connect its relay](connect-a-relay.md). The enabled ones are listed publicly at `GET /v1/gateway-clients`. Credentials with `gc:read` and `gc:write` can manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients). Every change is recorded in the [audit log](../operating/audit-log.md). +Register each number that relays SMS to this server, so the RelaySMS app lists it, then [connect its relay](connect-a-relay.md). Credentials with `gc:read` and `gc:write` can also manage them over the [REST API](../reference/rest-api.md#managing-gateway-clients); every change is recorded in the [audit log](../operating/audit-log.md). ## Commands @@ -17,7 +17,7 @@ A gateway client is a phone number that relays the SMS people send to this serve | `gateway-clients operators --country ` | Lists the operators of that country's gateway clients | | `gateway-clients mcc-mnc list` | Searches the PLMN table; filter with `--country-code`, `--network` or `--iso` | -Prefix each with `sudo ./publisher.sh`. `` is comma-separated, such as `https,smtp`. For example: +Prefix each with `sudo ./publisher.sh`. `` is how the relay reaches the Publisher, comma-separated: `https` for one that posts to the REST API, `smtp` for one that emails. For example: ```bash sudo ./publisher.sh gateway-clients create --msisdn +237600000000 --protocols https @@ -29,7 +29,7 @@ sudo ./publisher.sh gateway-clients create --msisdn +237600000000 --protocols ht `phonenumbers` gives the number's country, region and carrier name. The carrier is matched against that region's operators in `publisher/gateway_clients/mcc_mnc_table.json`, a vendored snapshot of [musalbas/mcc-mnc-table](https://github.com/musalbas/mcc-mnc-table). A territory without rows of its own, like Guernsey, uses its calling code's main region. -Names are compared word by word, ignoring case, accents, company-type words ("Telecom", "Ltd", "Mobile") and the country's name, so "MTN Cameroon" matches "MTN" and "Beeline" matches "Bee Line/Unitel". Words many of the region's networks share count only weakly. +Names are compared word by word, ignoring case, accents, the country's name and words such as "Telecom" or "Ltd". | `match` | Candidates | | --- | --- | @@ -37,4 +37,4 @@ Names are compared word by word, ignoring case, accents, company-type words ("Te | `region` | Every operator in the country, because the carrier is unknown (common for US and Canadian numbers) or matched nothing | | `none` | None: the number couldn't be placed. Enter the details yourself. | -A PLMN missing from the snapshot can still be entered with `--operator-code`. +When `suggest` can't place the operator, as for many US, Canadian and Twilio numbers, pass `--country` and `--operator` yourself, or leave the operator out. The public list shows each number's country and operator. diff --git a/docs/installation/celery-broker.md b/docs/installation/celery-broker.md index 0472836..faad0a3 100644 --- a/docs/installation/celery-broker.md +++ b/docs/installation/celery-broker.md @@ -12,7 +12,7 @@ The REST API and the email listener queue publications for the Celery worker thr ## RabbitMQ -A setup script installs RabbitMQ, creates a vhost and user, and writes the broker settings to `.env`: +A setup script installs RabbitMQ, creates a vhost and user, and writes the broker settings to `.env`. During install: diff --git a/docs/installation/database.md b/docs/installation/database.md index 7eb25be..4b272df 100644 --- a/docs/installation/database.md +++ b/docs/installation/database.md @@ -6,11 +6,11 @@ | MySQL 8.0.28+ or MariaDB | Heavier load, or a database on another host | The server's own (TDE) | | PostgreSQL 12+ | Heavier load, or a database on another host | The disk's | -Sensitive fields are also encrypted in every dialect; see [Encryption keys](../configuration/encryption-keys.md). +In every dialect, the server's private keys are encrypted, and so are platform tokens while field encryption is on; see [Encryption keys](../configuration/encryption-keys.md). -## MySQL, MariaDB or PostgreSQL +## Database servers -A setup script installs the server, creates the database and user, and writes the connection details to `.env`. Running it again without `--db-password` gives the user a new password. The MySQL script installs `mysql-server`, which only Ubuntu ships; on Debian, set up MariaDB yourself and pass `--db-existing`. +A setup script installs the server, creates the database and user, and writes the connection details to `.env`. Running it again without `--db-password` gives the user a new password. On Debian, install MariaDB yourself and pass `--db-existing`. During install: diff --git a/docs/installation/docker.md b/docs/installation/docker.md index b7cbb5c..d7b16ee 100644 --- a/docs/installation/docker.md +++ b/docs/installation/docker.md @@ -2,8 +2,6 @@ `docker-compose.yml` runs every service in one container, with its data in `./data` and its settings in `.env` (see the [settings reference](../configuration/settings.md)). It needs the tools in the [quick start's requirements](../getting-started/quick-start.md#requirements). -To install: - ```bash git clone --recurse-submodules https://github.com/smswithoutborders/RelaySMS-Publisher.git cd RelaySMS-Publisher @@ -13,12 +11,12 @@ docker compose up -d --build The container applies pending migrations, then starts every service. It publishes `PORT` (16000) and `GRPC_PORT` (6000) on every interface. To serve it publicly, bind them to `127.0.0.1` in `docker-compose.yml` and put [nginx with TLS](reverse-proxy.md#by-hand) in front. -Run the admin CLI with `docker compose exec publisher python -m publisher `. +Run the admin CLI with `docker compose exec publisher python -m publisher `. [Back up](../operating/backup-restore.md) `.env` and `./data`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. -## Updating +To update, see [Upgrading](../operating/upgrading.md#update-with-docker). -See [Upgrading](../operating/upgrading.md#update-with-docker). +## Next steps -Next: [Settings reference](../configuration/settings.md). +Continue with [After installing](server.md#after-installing) from step 3, running the CLI as `docker compose exec publisher python -m publisher`. diff --git a/docs/installation/manual.md b/docs/installation/manual.md index 24c9e65..50c8f4b 100644 --- a/docs/installation/manual.md +++ b/docs/installation/manual.md @@ -81,4 +81,6 @@ These are the steps the [installer](server.md) runs, for when you need to change systemctl enable --now relaysms-publisher.target ``` -Then set up [nginx and TLS](reverse-proxy.md) and continue with [After installing](server.md#after-installing). +## Next steps + +Set up [nginx and TLS](reverse-proxy.md#by-hand), then continue with [After installing](server.md#after-installing). diff --git a/docs/installation/multiple-instances.md b/docs/installation/multiple-instances.md index dbde213..d1829bd 100644 --- a/docs/installation/multiple-instances.md +++ b/docs/installation/multiple-instances.md @@ -2,13 +2,13 @@ To run a second Publisher on the same host, give it its own install directory, instance name and ports: -1. Install it without nginx. When it shares a database or RabbitMQ server with the first instance, also pass `--db-name acme --db-user acme` or `--broker-vhost acme --broker-user acme`; reusing the first instance's user would give it a new password and break the first instance: +1. Install it without nginx. When it shares a database or RabbitMQ server with the first instance, give it its own user: pass `--db-name acme --db-user acme` or `--broker-vhost acme --broker-user acme`. ```bash sudo ./install.sh --install-dir /srv/relaysms-acme --instance-name acme --skip-nginx ``` -2. Set free `PORT` and `GRPC_PORT` values in `/srv/relaysms-acme/.env`. The installer gives every instance 16000 and 6000. +2. Set `PORT` and `GRPC_PORT` in `/srv/relaysms-acme/.env` to ports no other instance uses. 3. Restart it, then put nginx in front: diff --git a/docs/installation/requirements.md b/docs/installation/requirements.md index af3f4ef..d9ed2c4 100644 --- a/docs/installation/requirements.md +++ b/docs/installation/requirements.md @@ -2,9 +2,9 @@ - **Operating system**: Ubuntu 24.04 or Debian 13, or any Linux host with Docker. - **Python**: 3.12 or newer. The installer installs it, with the other system packages and Rust. -- **Database**: SQLite (the default), MySQL 8.0.28 or newer, MariaDB, or PostgreSQL 12 or newer. See [Database](database.md). +- **Database**: SQLite (the default) or a database server. See [Database](database.md) for the supported ones. - **Domain**: a DNS name pointing at the server, for the TLS certificate. See [Reverse proxy and TLS](reverse-proxy.md). - **Inbound ports (firewall)**: 80 and 443 for nginx. On a server install, the services listen on `127.0.0.1` only. Docker publishes `PORT` (16000) and `GRPC_PORT` (6000). - **Outbound access**: `github.com`, `pypi.org`, the Rust installer and the system's package mirrors to install the Publisher and its adapters; Let's Encrypt for certificates; each platform's API to publish. - **Platform credentials**: from each platform you offer, such as a Google Cloud OAuth client for Gmail. See [Provider guides](../platforms/manage.md#provider-guides). -- **Gateway client**: at least one phone number that relays SMS, on an Android phone running Deku SMS or a Twilio number. See [Connecting a relay](../gateway-clients/connect-a-relay.md). +- **Gateway client**: at least one phone number that relays SMS, such as an Android phone running Deku SMS. See [Connecting a relay](../gateway-clients/connect-a-relay.md). diff --git a/docs/installation/reverse-proxy.md b/docs/installation/reverse-proxy.md index 285d870..32798be 100644 --- a/docs/installation/reverse-proxy.md +++ b/docs/installation/reverse-proxy.md @@ -1,14 +1,14 @@ # Reverse proxy and TLS -The services listen on `127.0.0.1`. nginx serves them on port 443 with a Let's Encrypt certificate, from [the site template](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/deploy/nginx/relaysms-publisher-nginx.conf.template). It sends `/publisher.v3.Publisher` and `/grpc.health.v1.Health` to gRPC on `GRPC_PORT`, and everything else to REST on `PORT`, and rate-limits logins and requests carrying HTTP Basic credentials. +The services listen on `127.0.0.1`. nginx serves them on port 443 with a Let's Encrypt certificate, from [the site template](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/deploy/nginx/relaysms-publisher-nginx.conf.template). It sends `/publisher.v3.Publisher` and `/grpc.health.v1.Health` to gRPC on `GRPC_PORT`, and everything else to REST on `PORT`. It rate-limits logins and HTTP Basic requests ([limits](../operating/credentials.md#logging-in)). ## With the installer -Pass `--site-name DOMAIN` to the [installer](server.md#run-the-installer). Running it again leaves an existing site file alone, and skips Certbot when the domain already has a certificate; use [`manage.sh nginx`](#on-an-existing-install) to render the site again. +Pass `--site-name ` to the [installer](server.md#run-the-installer). Running it again leaves an existing site file alone, and skips Certbot when the domain already has a certificate; use [`manage.sh nginx`](#on-an-existing-install) to render the site again. ## On an existing install -`manage.sh nginx` renders the site from the template, obtains or reattaches its certificate, and turns on HTTP/2, which gRPC needs. Run it to add nginx later, or after an update that changes the template. It keeps the previous file as `.conf.bak` and restores it if the new one fails. +`manage.sh nginx` renders the site from the template, obtains or reattaches its certificate, and turns on HTTP/2, which gRPC needs. Run it to add nginx later, or after an update that changes the template. It keeps the previous file as `.conf.bak` and restores it if the new one fails. ```bash sudo apt-get install -y nginx certbot python3-certbot-nginx diff --git a/docs/installation/server.md b/docs/installation/server.md index 9c83ad4..1fe26ea 100644 --- a/docs/installation/server.md +++ b/docs/installation/server.md @@ -4,6 +4,9 @@ The installer sets up the Publisher on Ubuntu 24.04 or Debian 13 under systemd, ## Run the installer +> [!TIP] +> For a dedicated service user with no login, run the installer from a root shell (`sudo -i`). The service user can't be changed later without reinstalling. + ```bash curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ | sudo bash @@ -26,15 +29,15 @@ curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publishe | Flag | Does | | --- | --- | -| `--site-name DOMAIN` | Puts nginx with a Let's Encrypt certificate in front ([details](reverse-proxy.md)) | -| `--letsencrypt-email EMAIL` | Email for Let's Encrypt renewal notices | -| `--docs-site-name DOMAIN` | Serves this manual on its own domain ([details](#serving-the-manual)) | +| `--site-name ` | Puts nginx with a Let's Encrypt certificate in front ([details](reverse-proxy.md)) | +| `--letsencrypt-email ` | Email for Let's Encrypt renewal notices | +| `--docs-site-name ` | Serves this manual on its own domain ([details](#serving-the-manual)) | | `--skip-nginx` | Skips nginx and certificates entirely | | `--setup-db mysql\|postgres` | Uses MySQL or PostgreSQL instead of SQLite ([details](database.md)) | | `--setup-broker rabbitmq` | Uses RabbitMQ instead of SQLite as the Celery broker ([details](celery-broker.md)) | | `--setup-observability` | Adds SigNoz and Uptime Kuma ([details](../operating/observability.md)) | -| `--install-dir PATH`, `--instance-name NAME` | Installs elsewhere, or a second copy on the same host ([details](multiple-instances.md)) | -| `--branch BRANCH` | Installs another branch than `main` | +| `--install-dir `, `--instance-name ` | Installs elsewhere, or a second copy on the same host ([details](multiple-instances.md)) | +| `--branch ` | Installs another branch than `main` | `install.sh --help` lists every flag, including those for an existing database or broker. @@ -49,21 +52,6 @@ The installer: Running it again updates the install in place and keeps its `.env`. Run it as the same user as the first time; it refuses another service user. -## Serving the manual - -`--docs-site-name` builds this manual into `book/` and serves it with nginx on its own domain and certificate. `manage.sh update` rebuilds it. - -```bash -curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ - | sudo bash -s -- --site-name publisher.example.com --docs-site-name docs.publisher.example.com -``` - -On an install that's already running, add it from the install directory: - -```bash -sudo ./manage.sh docs docs.publisher.example.com -``` - ## After installing 1. Go to the install directory and check that the services are up: @@ -76,7 +64,7 @@ sudo ./manage.sh docs docs.publisher.example.com You should see `{"status":"ok"}`, also at `https:///health` once nginx is set up. -2. Set what you need in `.env` (see the [settings reference](../configuration/settings.md)), then check it and restart: +2. The defaults suit one server behind nginx. Change `.env` only for what they don't cover, such as a MySQL or PostgreSQL database ([settings reference](../configuration/settings.md)), then check it and restart: ```bash sudo ./manage.sh check @@ -92,8 +80,24 @@ sudo ./manage.sh docs docs.publisher.example.com > [!IMPORTANT] > The password is printed only once. -4. Back up `.env`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. +4. [Back up](../operating/backup-restore.md) `.env`: its [encryption keys](../configuration/encryption-keys.md) are the only way to read the data. + +## Serving the manual + +`--docs-site-name` builds this manual into `book/` and serves it with nginx on its own domain and certificate. `manage.sh update` rebuilds it. + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh \ + | sudo bash -s -- --site-name publisher.example.com --docs-site-name docs.publisher.example.com +``` + +On an install that's already running, add it from the install directory: + +```bash +sudo ./manage.sh docs docs.publisher.example.com +``` -5. [Add the platform adapters](../platforms/manage.md) you want to offer, and [register the gateway clients](../gateway-clients/manage.md) that relay SMS to this server. +## Next steps -Next: [Services](../operating/services.md). +- [Add a platform adapter](../platforms/manage.md), then follow its [provider guide](../platforms/manage.md#provider-guides). +- [Register a gateway client](../gateway-clients/manage.md) and [connect its relay](../gateway-clients/connect-a-relay.md). diff --git a/docs/operating/backup-restore.md b/docs/operating/backup-restore.md index 1cc5316..0089f0f 100644 --- a/docs/operating/backup-restore.md +++ b/docs/operating/backup-restore.md @@ -7,7 +7,9 @@ A backup is: - **A dump of the database**, with MySQL or PostgreSQL. - **Any file `.env` points to outside `data/`**, such as `GRPC_TLS_CERT_FILE` or `SMTP_TLS_CLIENT_KEY`. -The installer recreates the nginx sites, certificates and service units, so `/etc` needs no backup. +The installer recreates the service units, and the nginx sites and certificates for the domains you pass it, so `/etc` needs no backup. If you run the [Twilio relay](../gateway-clients/connect-a-relay.md#twilio), back up its `.env` too: its `SENDER_HASH_KEY` keeps senders' hashes the same. + +Copy the archives off the host, and test a restore on a spare machine now and then. > [!CAUTION] > Store the `.env` archive apart from the data. Anyone with both can read the linked accounts' tokens. @@ -86,4 +88,4 @@ For a database on another host, or named differently in `.env`, connect with the ## Next steps -[Check that the services are up](services.md#is-it-working). +After a restore, [check that the services are up](services.md#is-it-working). diff --git a/docs/operating/credentials.md b/docs/operating/credentials.md index 1ded97b..abe4d2a 100644 --- a/docs/operating/credentials.md +++ b/docs/operating/credentials.md @@ -49,6 +49,6 @@ Prefix each with `sudo ./publisher.sh`. Credentials with `creds:write` can do th - **Web session:** `POST /v1/auth/login`, then send the session cookie with every request (`credentials: "include"` in `fetch`). Logins and writes (`POST`, `PATCH`, `DELETE`) with the cookie must come from the API's own origin or one listed in `AUTH_WEB_ORIGINS`, on the same site as the API. A session ends after `AUTH_SESSION_IDLE_MINUTES` idle (30) or `AUTH_SESSION_MAX_HOURS` (12). - **HTTP Basic:** the username and password on every request, such as `curl -u analyst: https:///v1/stats/publications`. -Scope changes apply on the credential's next request. nginx allows 5 logins a minute per IP address, and 30 HTTP Basic requests a minute per IP address to `/v1/stats`, `/v1/auth` and `/v1/creds`. +Scope changes apply on the credential's next request. nginx allows 5 logins a minute per IP address, and 30 HTTP Basic requests a minute per IP address. Settings: [REST API access](../configuration/settings.md#rest-api-access). diff --git a/docs/operating/observability.md b/docs/operating/observability.md index ce707b4..fa97842 100644 --- a/docs/operating/observability.md +++ b/docs/operating/observability.md @@ -1,6 +1,6 @@ # Observability -Self-hosted tracing, metrics and log export (SigNoz) and uptime monitoring (Uptime Kuma) for the Publisher's five services. Both are optional; tracing stays off while `OTEL_EXPORTER_OTLP_ENDPOINT` is blank. +Self-hosted tracing, metrics and log export (SigNoz) and uptime monitoring (Uptime Kuma) for the Publisher's services. Both are optional; tracing, metrics and log export stay off while `OTEL_EXPORTER_OTLP_ENDPOINT` is blank. ## Set up @@ -149,7 +149,7 @@ docker compose \ until curl -fs http://127.0.0.1:8090/api/v1/version | grep -q setupCompleted; do sleep 2; done curl -X POST http://127.0.0.1:8090/api/v1/register \ -H "Content-Type: application/json" \ - -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@relaysms.local","password":""}' + -d '{"name":"Admin","orgId":"","orgName":"RelaySMS","email":"admin@example.com","password":""}' ``` The template stores SigNoz's metadata in Postgres (`metastore.kind: postgres`); set `sqlite` for a single node. diff --git a/docs/operating/security-hardening.md b/docs/operating/security-hardening.md index 39721d2..b83a69e 100644 --- a/docs/operating/security-hardening.md +++ b/docs/operating/security-hardening.md @@ -2,7 +2,7 @@ A server install listens on `127.0.0.1` only, runs each service under systemd sandboxing with everything read-only except its data directories, and sits behind nginx with TLS and rate limits. These steps close what's left to you. -## Network +## Network and firewall 1. Allow only SSH, HTTP and HTTPS in: diff --git a/docs/operating/services.md b/docs/operating/services.md index d52a71f..9799aea 100644 --- a/docs/operating/services.md +++ b/docs/operating/services.md @@ -1,6 +1,6 @@ # Services -The Publisher runs as five systemd services, grouped under `relaysms-publisher.target`: +The Publisher runs as systemd services, grouped under `relaysms-publisher.target`: | Service | Runs | | --- | --- | @@ -40,11 +40,11 @@ Run it from the install directory with `sudo`: | `update [--migrate]` | Pulls the latest code, rebuilds and restarts the services ([details](upgrading.md)) | | `migrate` | Applies database migrations | | `check` | Reports every missing or invalid setting | -| `nginx [DOMAIN]` | Renders the nginx site again and reattaches or obtains its certificate ([details](../installation/reverse-proxy.md#on-an-existing-install)) | -| `docs [DOMAIN]` | Builds this manual and serves it on its own domain ([details](../installation/server.md#serving-the-manual)) | +| `nginx []` | Renders the nginx site again and reattaches or obtains its certificate ([details](../installation/reverse-proxy.md#on-an-existing-install)) | +| `docs []` | Builds this manual and serves it on its own domain ([details](../installation/server.md#serving-the-manual)) | | `uninstall` | Removes the services and the install directory, with `.env` and `data/` | -`uninstall` leaves the nginx sites, certificates, database and broker servers, and the service user. +`uninstall` keeps the database and broker servers and the service user. `logs` takes: @@ -59,8 +59,6 @@ For example: sudo ./manage.sh logs --unit rest --unit grpc --since "1 hour ago" --lines 200 ``` -When `.env` has an invalid setting, `update` still restarts every service, then exits with an error; services that use that setting fail to start ([fix](troubleshooting.md#update-ends-with-env-has-errors)). See [Upgrading](upgrading.md). - ## publisher.sh `./publisher.sh` runs the admin CLI as the service user, with the virtualenv and `.env`: @@ -78,7 +76,7 @@ Use it instead of `python3 -m publisher`: run as another user, the CLI can leave | Path | Holds | | --- | --- | | `/opt/relaysms/relaysms-publisher/` | The install | -| `.env` | Settings (`root:`, mode 640) | +| `.env` | Settings (`root:`, mode 640) | | `data/` | SQLite database, Celery files and adapters (owned by the service user, mode 750) | | `/etc/systemd/system/relaysms-publisher*` | Service units | -| `/etc/nginx/sites-available/.conf` | nginx sites | +| `/etc/nginx/sites-available/.conf` | nginx sites | diff --git a/docs/operating/troubleshooting.md b/docs/operating/troubleshooting.md index e0e889c..e3ac2ea 100644 --- a/docs/operating/troubleshooting.md +++ b/docs/operating/troubleshooting.md @@ -6,7 +6,7 @@ Start with the logs of the service involved: sudo ./manage.sh logs --unit --since "1 hour ago" ``` -`` is `rest`, `grpc`, `worker`, `beat` or `smtp`. +`` is one of the [units `logs` takes](services.md#managesh). ## Services @@ -40,7 +40,7 @@ The container applies migrations before the services start. Wait, or follow `doc | --- | --- | | `Published via ''.` | It worked | | `Failed to process payload: ...` | The payload didn't decrypt or validate, its keys are unknown or used up, or offline publishing refused it ([Offline publishing](../configuration/offline-publishing.md)) | -| `Failed to publish message: Adapter error: ...` | The platform refused it; the rest of the line is the adapter's reason | +| `Failed to publish message: Adapter error: ...` | The platform refused it; the rest of the line is the adapter's reason. Fix it with the platform's [provider guide](../platforms/manage.md#provider-guides), then the user sends the same SMS again. | | `Session can't be assembled from segment(s): ...` | A multi-part payload's segments don't fit together | | `An unexpected error occurred during task processing.` | A bug; the traceback follows. [Report it](https://github.com/smswithoutborders/RelaySMS-Publisher/issues). | | No line at all | The worker isn't running (`sudo ./manage.sh status`), or a multi-part payload is still waiting for its other segments, for up to `PAYLOAD_SESSION_MAX_AGE_HOURS` | @@ -51,6 +51,23 @@ The container applies migrations before the services start. Wait, or follow `doc Check the adapter shows ✓ under Configured in `sudo ./publisher.sh platforms list`, then see its [provider guide](../platforms/manage.md#provider-guides)'s troubleshooting. +## Relay + +### An SMS was sent, but the worker logs nothing + +Check that the relay received it: + +- With Deku SMS, the phone is online and Deku SMS is its default SMS app. +- With Twilio, follow the relay's log (`sudo journalctl -u relaysms-gatewayclient-twilio`) and Twilio's error log, under Monitor > Logs > Errors in the Twilio console. + +### The Twilio relay answers `403` `Invalid Twilio signature.` + +The webhook URL in the Twilio console doesn't match the URL the relay sees, scheme and host included. Set it to the relay's exact `https://` URL. + +### The Twilio relay answers `502` `Forwarding failed.` + +A `FORWARD_URLS_JSON` URL refused the payload or couldn't be reached; the relay's log names it. Check that `https:///v1/publications` answers from the relay's host. + ## REST API ### `Invalid username or password.` @@ -71,7 +88,7 @@ The web client's origin is neither the API's own nor listed in `AUTH_WEB_ORIGINS ### `Too many requests. Please try again shortly.` -nginx allows 5 logins a minute per IP address, and 30 HTTP Basic requests a minute per IP address to `/v1/stats`, `/v1/auth` and `/v1/creds`. Wait a minute and try again. +nginx rate-limits logins and HTTP Basic requests ([limits](credentials.md#logging-in)). Wait a minute and try again. ## nginx diff --git a/docs/operating/upgrading.md b/docs/operating/upgrading.md index 3ebee08..4bef4aa 100644 --- a/docs/operating/upgrading.md +++ b/docs/operating/upgrading.md @@ -1,14 +1,14 @@ # Upgrading -A server install follows the branch it was installed from, `main` by default. An update pulls that branch, rebuilds, and restarts the services. +A server install follows the branch it was installed from, `main` by default. ## Before you upgrade 1. [Back up](backup-restore.md). Migrations can't be undone, so going back to older code needs the backup. -2. Note the current commit, to roll back to: +2. From the install directory, note the current commit, to roll back to: ```bash - git rev-parse HEAD + sudo git rev-parse HEAD ``` ## Update a server install @@ -22,18 +22,20 @@ sudo ./manage.sh check sudo ./manage.sh restart ``` -1. `update --migrate` stops the services, pulls the code, reinstalls the dependencies, rebuilds, applies pending migrations and starts the services again. With no migrations pending, `--migrate` changes nothing. When the install serves this manual, it rebuilds it too. +1. `update --migrate` stops the services, pulls the code, reinstalls the dependencies, rebuilds, applies pending migrations and starts the services again. When the install serves this manual, it rebuilds it too. 2. `sync-env.sh` adds the settings the new `template.env` has and `.env` lacks, showing the change before writing it. 3. `check` reports any setting still missing or invalid; fill it in `.env`. 4. `restart` loads the new settings. -If `git diff --stat ORIG_HEAD HEAD -- deploy/nginx` lists changes, render the site again: +When `.env` has an invalid setting, `update` still restarts every service, then exits with an error ([fix](troubleshooting.md#update-ends-with-env-has-errors)). + +If the update changed the nginx template, which `sudo git diff --stat ORIG_HEAD HEAD -- deploy/nginx` lists, render the site again: ```bash sudo ./manage.sh nginx ``` -If you edited tracked files that the update changes, `update` stops before pulling and leaves the services stopped. Discard the edits or `git stash` them, then run `update` again. +If you edited tracked files that the update changes, `update` refuses to pull. Discard the edits or `git stash` them, then run `update` again. ## Roll back @@ -46,7 +48,7 @@ If you edited tracked files that the update changes, `update` stops before pulli sudo ./manage.sh update --pulled ``` -To follow the branch again later, run `sudo git checkout main`, then `sudo ./manage.sh update --migrate`. +To follow the branch again later, run `sudo git checkout `, then `sudo ./manage.sh update --migrate`. ## Update with Docker @@ -71,10 +73,14 @@ See [Versions](../platforms/manage.md#versions). Steps a particular upgrade needs once, newest first. -### Gateway clients moved to the database +### Gateway client registry import (migration 019) + +For installs from before 2026-10-07. `update --migrate` applies migration 019, which imports `data/gateway_clients/registry.json` (or `GATEWAY_CLIENTS_REGISTRY_FILE`, if `.env` sets it). It logs the entries in `mcc_mnc_overrides.json` without importing them. Delete both files afterwards. -### Adapters moved to the database +### Adapter registry import + +For installs from before 2026-10-06. -`update --migrate` and `manage.sh migrate` register adapters installed before the adapter registry moved to the database, and move their settings and files out of the code. The Docker container does it at every start. +`update --migrate` and `manage.sh migrate` register adapter directories on disk that the database lacks, and move their settings and files out of the code. The Docker container does it at every start. diff --git a/docs/platforms/manage.md b/docs/platforms/manage.md index c3f0d37..eda6e29 100644 --- a/docs/platforms/manage.md +++ b/docs/platforms/manage.md @@ -1,6 +1,6 @@ # Managing adapters -Each platform (Gmail, Telegram, Mastodon, ...) is served by an adapter installed from its own GitHub repository. Its files live under `data/platforms/` (see [Files](#files)). +Each platform, such as Gmail, is served by an adapter installed from its own GitHub repository. Run these commands from the install directory with [`publisher.sh`](../operating/services.md#publishersh), which runs them as the service user. @@ -11,13 +11,13 @@ Run these commands from the install directory with [`publisher.sh`](../operating | `platforms add [--tag ]` | Installs the newest version tag, or `` | | `platforms update [] [--tag ]` | Moves one adapter, or all, to a version tag | | `platforms list` | Lists every adapter, disabled ones included | -| `platforms disable ` | Hides it from users; it still revokes tokens | +| `platforms disable ` | Hides it from users; linked accounts can still be unlinked | | `platforms enable ` | Offers it again | | `platforms remove [--force]` | Uninstalls it ([details](#removing)) | | `platforms config show ` | Lists its settings, without secret values | | `platforms config set =\|` | Changes settings; a bare `` asks for its value | | `platforms setup [--input =]` | Runs the adapter's setup and saves its settings | -| `platforms import` | Registers adapters installed before the database registry ([details](../operating/upgrading.md#adapters-moved-to-the-database)) | +| `platforms import` | Registers adapter directories on disk that the database lacks, and moves their files out of the code ([details](../operating/upgrading.md#adapter-registry-import)) | | `platforms exec -- ` | Runs a command the adapter installs | Prefix each with `sudo ./publisher.sh`, for example: @@ -68,11 +68,9 @@ sudo ./publisher.sh platforms config set rmail --unset alias_prefix sudo ./publisher.sh platforms setup gmail --input client_file=@/tmp/client_secret.json ``` -Once an adapter is configured, link a test account from the RelaySMS app to check it end to end. - ## Redirect URIs -OAuth2 adapters (Gmail, Mastodon, Bluesky) send the user back to a redirect URI after they authorize. The client app chooses it and sends it to the Publisher when it links an account; the adapter uses the first of its `redirect_uris` setting when the app sends none. +OAuth2 adapters, such as Gmail, send the user back to a redirect URI after they authorize. The client app chooses it and sends it to the Publisher when it links an account; the adapter uses the first of its `redirect_uris` setting when the app sends none. 1. Run an HTTPS page that receives the authorization `code` and `state` and hands them back to the app. 2. Configure the apps to send that page's URI. @@ -97,6 +95,7 @@ An adapter may install admin commands. `exec` runs one from the adapter's virtua sudo ./publisher.sh platforms exec -- --help ``` -## Writing an adapter +## Next steps -See [Writing an adapter](../developers/writing-an-adapter.md). +- Follow the [provider guide](#provider-guides) for each platform you offer, then link a test account from the RelaySMS app. +- To support another platform, see [Writing an adapter](../developers/writing-an-adapter.md). diff --git a/docs/platforms/relaysms-mail.md b/docs/platforms/relaysms-mail.md index 1cbfc70..273b4c2 100644 --- a/docs/platforms/relaysms-mail.md +++ b/docs/platforms/relaysms-mail.md @@ -20,7 +20,7 @@ The `rmail` adapter gives each user an email alias tied to their phone number, a ## SMTP -The adapter sends each message from your mailbox address over SMTP, and SimpleLogin relays it from the user's alias. Port 465 uses TLS; any other port uses STARTTLS when the server offers it. +The adapter sends each message from your mailbox address over SMTP, and SimpleLogin relays it from the user's alias. For a Gmail mailbox, use `smtp.gmail.com`, port `465`, the Gmail address, and an [app password](https://myaccount.google.com/apppasswords), which needs 2-Step Verification on the account. diff --git a/docs/platforms/telegram.md b/docs/platforms/telegram.md index 420f4ab..8c973ee 100644 --- a/docs/platforms/telegram.md +++ b/docs/platforms/telegram.md @@ -1,6 +1,6 @@ # Telegram -Lets users send Telegram messages. Users link with their phone number, a login code and, if they set one, their two-step verification password. The adapter needs a Telegram API ID and hash. +Lets users send Telegram messages. Users link with their phone number, a login code and, if they set one, their two-step verification password. *Last checked against my.telegram.org: 2026-10-22.* diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 780e1e7..020fe3c 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -14,7 +14,7 @@ Run them with `sudo`; all but `install.sh` from the install directory. Each take ## manage.sh -`status`, `logs`, `start`, `stop`, `restart`, `enable`, `disable`, `update`, `migrate`, `check`, `nginx`, `docs`, `uninstall`. See [Services](../operating/services.md#managesh). +See [Services](../operating/services.md#managesh). ## publisher.sh diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 1c3c0c8..aadcbf4 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -3,19 +3,19 @@ - **Adapter** (platform adapter): the plugin that publishes to one platform, such as Gmail, installed from its own repository. See [Managing adapters](../platforms/manage.md). - **Administrator**: a credential holding every scope. - **Audit event**: a record of a change or login. See [Audit log](../operating/audit-log.md). -- **Category**: what an adapter publishes: email, message, text or bridge (`--cat-id`). +- **Category**: what an adapter publishes, such as email (`--cat-id`). - **Credential**: a username and password that logs in to the REST API's admin endpoints, with its own scopes. See [Credentials and access](../operating/credentials.md). - **Ephemeral key pool**: the client public keys and server key pairs each linked account holds, up to 256 of each; each pair is used once, with an identity key, by a payload or a token check. See [Security model](../explanation/security-model.md). -- **Gateway client**: a phone number that relays SMS to the Publisher, from an Android phone running Deku SMS or from Twilio. See [Connecting a relay](../gateway-clients/connect-a-relay.md). +- **Gateway client**: a phone number that relays SMS to the Publisher, such as an Android phone running Deku SMS. See [Connecting a relay](../gateway-clients/connect-a-relay.md). - **Identity key**: one of the server's 256 long-term X25519 keys, used in every gRPC request and payload, and alone for offline payloads. - **Linked account**: a user's account on a platform, authorized once while online. The Publisher stores its token encrypted. - **MSISDN**: a phone number in international format, such as `+237600000000`. -- **OAuth2**: linking through the platform's consent page, as Gmail, Mastodon and Bluesky do. -- **Offline payload**: content sent without a linked account, published through RelaySMS Mail; the whitepaper calls these offline-first publications. See [Offline publishing](../configuration/offline-publishing.md). +- **OAuth2**: linking through the platform's consent page, as for Gmail. +- **Offline payload**: content sent without a linked account, published through [RelaySMS Mail](../platforms/relaysms-mail.md). See [Offline publishing](../configuration/offline-publishing.md). - **Payload**: the encrypted content the RelaySMS app sends by SMS; one carrying an attachment arrives in several segments. - **PLMN**: a mobile network, identified by its MCC (country code) and MNC (network code). See [How suggest works](../gateway-clients/manage.md#how-suggest-works). -- **PNBA** (phone number-based authentication): linking with a code sent to the user's phone, as Telegram and RelaySMS Mail do. -- **Protocol**: how an adapter links accounts, `oauth2` or `pnba` (`--proto-id`). For a payload, how it reached the Publisher: `https` (the REST API) or `smtp` (the email listener). +- **PNBA** (phone number-based authentication): linking with a code sent to the user's phone, as for Telegram. +- **Protocol**: how an adapter links accounts, such as `oauth2` (`--proto-id`). For a payload, how it reached the Publisher, such as `https` (the REST API); see [Offline publishing](../configuration/offline-publishing.md). - **Redirect URI**: where an OAuth2 platform sends the user back after they authorize. See [Redirect URIs](../platforms/manage.md#redirect-uris). - **rmail**: the name of the RelaySMS Mail adapter. See [RelaySMS Mail](../platforms/relaysms-mail.md). - **Scope**: a permission a credential holds, such as `gc:write`. See [Scopes](../operating/credentials.md#scopes). diff --git a/docs/reference/grpc-api.md b/docs/reference/grpc-api.md index ea17569..30dbc68 100644 --- a/docs/reference/grpc-api.md +++ b/docs/reference/grpc-api.md @@ -37,27 +37,27 @@ The examples use `-plaintext` against a local server; drop it for a server behin ### Get OAuth2 authorization URL -Generates an OAuth2 authorization URL for initiating the OAuth2 flow. +Returns the URL that starts the OAuth2 flow. **Request:** `GetOAuth2AuthorizationUrlRequest` | Field | Type | Required | Description | | :--- | :--- | :--- | :--- | | platform | string | Yes | Platform identifier, such as `gmail` | -| state | string | Optional | OAuth2 `state`, returned unchanged | -| code_verifier | string | Optional | PKCE code verifier | -| autogenerate_code_verifier | bool | Optional | Auto-generate code verifier if not provided | -| redirect_url | string | Optional | Redirect URL for OAuth2 application | -| request_identifier | string | Optional | Request identifier for tracking | +| state | string | No | OAuth2 `state`, returned unchanged | +| code_verifier | string | No | PKCE code verifier | +| autogenerate_code_verifier | bool | No | Generate a code verifier when none is sent | +| redirect_url | string | No | Redirect URL for OAuth2 application | +| request_identifier | string | No | Request identifier for tracking | **Response:** `GetOAuth2AuthorizationUrlResponse` | Field | Type | Description | | :--- | :--- | :--- | | authorization_url | string | Generated authorization URL | -| state | string | State parameter from request | -| code_verifier | string | Code verifier used in PKCE flow | -| message | string | Response message | +| state | string | The `state` sent | +| code_verifier | string | PKCE code verifier used | +| message | string | Human-readable result | | scope | string | Authorization request scope | | client_id | string | OAuth2 application client ID | | redirect_url | string | OAuth2 application redirect URL | @@ -66,11 +66,11 @@ Generates an OAuth2 authorization URL for initiating the OAuth2 flow. ```bash grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ -d '{ "platform": "gmail", "autogenerate_code_verifier": true @@ -89,12 +89,12 @@ Exchanges an OAuth2 authorization code for tokens and stores them. Returns encry | :--- | :--- | :--- | :--- | | platform | string | Yes | Platform identifier | | authorization_code | string | Yes | OAuth2 authorization code | -| code_verifier | string | Optional | Code verifier for PKCE | -| redirect_url | string | Optional | Redirect URL for OAuth2 application | -| request_identifier | string | Optional | Request identifier for tracking | +| code_verifier | string | No | Code verifier for PKCE | +| redirect_url | string | No | Redirect URL for OAuth2 application | +| request_identifier | string | No | Request identifier for tracking | | client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 ephemeral public keys generated by the client, with key_id 0-255 each used once | -**PublicKey Message:** +**`PublicKey` message:** | Field | Type | Description | | :--- | :--- | :--- | @@ -105,8 +105,8 @@ Exchanges an OAuth2 authorization code for tokens and stores them. Returns encry | Field | Type | Description | | :--- | :--- | :--- | -| success | bool | Operation success | -| message | string | Response message | +| success | bool | `true` if the call succeeded | +| message | string | Human-readable result | | account_identifier | string | Account identifier, such as `mail@example.com` | | token_ciphertext | bytes | Encrypted token ciphertext | | token_id | uint32 | Unique identifier for the stored token | @@ -116,11 +116,11 @@ Exchanges an OAuth2 authorization code for tokens and stores them. Returns encry ```bash grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ -d @ \ -proto publisher.proto \ : publisher.v3.Publisher/ExchangeOAuth2CodeAndStore <' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ -d '{ "token_id": 2941967295, "key_id": 123 @@ -173,7 +173,7 @@ grpcurl -plaintext \ ### Get PNBA code -Requests a one-time passcode (OTP) for Phone Number-Based Authentication. +Requests a one-time passcode (OTP) for phone number-based authentication (PNBA). **Request:** `GetPNBACodeRequest` @@ -181,29 +181,29 @@ Requests a one-time passcode (OTP) for Phone Number-Based Authentication. | :--- | :--- | :--- | :--- | | platform | string | Yes | Platform identifier | | phone_number | string | Yes | Phone number to receive OTP | -| request_identifier | string | Optional | Request identifier for tracking | -| channel | string | Optional | Channel to send the code over, such as `whatsapp`, `telegram` or `signal` | +| request_identifier | string | No | Request identifier for tracking | +| channel | string | No | Channel to send the code over, such as `signal` | **Response:** `GetPNBACodeResponse` | Field | Type | Description | | :--- | :--- | :--- | -| success | bool | Operation success | -| message | string | Response message | +| success | bool | `true` if the call succeeded | +| message | string | Human-readable result | | expires_at | int64 | Optional. Code expiry, Unix epoch seconds | **Example:** ```bash grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ -d '{ "platform": "telegram", - "phone_number": "+1234567890" + "phone_number": "+237600000000" }' \ -proto publisher.proto \ : publisher.v3.Publisher/GetPNBACode @@ -220,17 +220,17 @@ Exchanges a one-time passcode (OTP) for a token, stores it, and returns encrypte | platform | string | Yes | Platform identifier | | phone_number | string | Yes | Phone number that received OTP | | authorization_code | string | Yes | PNBA authorization code | -| password | string | Optional | Password for two-step verification | -| request_identifier | string | Optional | Request identifier for tracking | +| password | string | No | Password for two-step verification | +| request_identifier | string | No | Request identifier for tracking | | client_ephemeral_public_keys | list[PublicKey] | Yes | Exactly 256 ephemeral public keys generated by the client, with key_id 0-255 each used once | -| channel | string | Optional | Channel the code was sent over, such as `whatsapp`, `telegram` or `signal` | +| channel | string | No | Channel the code was sent over, such as `signal` | **Response:** `ExchangePNBACodeAndStoreResponse` | Field | Type | Description | | :--- | :--- | :--- | -| success | bool | Operation success | -| message | string | Response message | +| success | bool | `true` if the call succeeded | +| message | string | Human-readable result | | two_step_verification_enabled | bool | Two-step verification status | | account_identifier | string | Account identifier | | token_ciphertext | bytes | Encrypted token ciphertext | @@ -242,17 +242,17 @@ Exchanges a one-time passcode (OTP) for a token, stores it, and returns encrypte ```bash grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ -d @ \ -proto publisher.proto \ : publisher.v3.Publisher/ExchangePNBACodeAndStore <' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ -d '{ "token_id": 2941967295, "key_id": 123 @@ -314,19 +314,19 @@ Replaces a token's key pools: the client sends 256 new ephemeral public keys and | Field | Type | Description | | :--- | :--- | :--- | -| success | bool | Operation success | -| message | string | Response message | +| success | bool | `true` if the call succeeded | +| message | string | Human-readable result | | server_ephemeral_public_keys | list[PublicKey] | Exactly 256 ephemeral public keys generated by the server | **Example:** ```bash grpcurl -plaintext \ - -H 'x-payload-bin: ' \ - -H 'x-public-key-bin: ' \ - -H 'x-key-id: ' \ - -H 'x-nonce-bin: ' \ - -H 'x-timestamp: ' \ + -H 'x-payload-bin: ' \ + -H 'x-public-key-bin: ' \ + -H 'x-key-id: ' \ + -H 'x-nonce-bin: ' \ + -H 'x-timestamp: ' \ -d @ \ -proto publisher.proto \ : publisher.v3.Publisher/SyncKeys < [!NOTE] -> Every RPC returns a standard [gRPC status code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html); `0 OK` is success. +Every RPC returns a standard [gRPC status code](https://grpc.github.io/grpc/core/md_doc_statuscodes.html); `0 OK` is success. diff --git a/docs/reference/rest-api.md b/docs/reference/rest-api.md index 3c4edf7..46bb892 100644 --- a/docs/reference/rest-api.md +++ b/docs/reference/rest-api.md @@ -1,15 +1,13 @@ # REST API -The REST API serves platform, gateway client and server key metadata, accepts encrypted payloads for publishing, and lets credentials read stats, manage credentials and read the audit log. +The REST API accepts payloads from gateway clients, serves public lists, and has the admin endpoints. The endpoint reference is [openapi.json](../openapi.json), generated from the app. Open it in any OpenAPI viewer, or set `API_DOCS_ENABLED=true` on a development server and browse `/docs`. -This page covers what the reference doesn't show well. - ## Base URL ``` -https:///v1 +https:///v1 ``` `/health` is the only endpoint outside `/v1`. @@ -48,13 +46,13 @@ List endpoints return `{"data": [...], "next": ..., "prev": ...}`, newest first. ## Publishing -`POST /v1/publications` takes JSON and tags payloads with protocol `https`. It needs no credentials: a payload publishes only if it decrypts with the keys of a linked account or, for an offline payload, passes the [offline publishing](../configuration/offline-publishing.md) rules. See [offline publishing](../configuration/offline-publishing.md) for how the protocol and `tag` decide whether offline payloads publish. +`POST /v1/publications` takes JSON and tags payloads with protocol `https`. It needs no credentials: a payload publishes only if it decrypts with the keys of a linked account or, for an offline payload, passes the [offline publishing](../configuration/offline-publishing.md) rules. It only checks the payload's structure before queueing it. Decryption, payload type and adapter errors happen later, so they're logged on the server, not returned. ## Errors -Error bodies are `{"error": ""}`. The message is meant to be shown to users; the server logs more detail. +Error bodies are `{"error": ""}`. The message is meant to be shown to users; the server logs more detail. | Status | Meaning | | :--- | :--- | @@ -66,5 +64,5 @@ Error bodies are `{"error": ""}`. The message is meant to be shown to u | `412 Precondition Failed` | `If-Match` doesn't match the current `ETag`; reload and retry | | `422 Unprocessable Entity` | Validation error | | `428 Precondition Required` | `If-Match` header missing | -| `429 Too Many Requests` | Rate limited (login and Basic auth) | +| `429 Too Many Requests` | Rate limited by nginx ([limits](../operating/credentials.md#logging-in)) | | `500 Internal Server Error` | Unexpected server error | diff --git a/publisher.sh b/publisher.sh index a346fb8..8378566 100755 --- a/publisher.sh +++ b/publisher.sh @@ -17,7 +17,7 @@ ENV_FILE="$INSTALL_DIR/.env" VENV_DIR="$INSTALL_DIR/venv" [ -x "$VENV_DIR/bin/python3" ] || - error "Virtualenv not found at $VENV_DIR. Run install.sh, or see Development in README.md." + error "Virtualenv not found at $VENV_DIR. Run install.sh, or see Development setup in CONTRIBUTING.md." INSTANCE_NAME="$(read_instance_name)" SERVICE_USER="$(detect_service_user)" diff --git a/template.env b/template.env index d8e2de2..6071082 100644 --- a/template.env +++ b/template.env @@ -1,8 +1,8 @@ # REST API HOST=127.0.0.1 PORT=16000 -# Worker processes for scripts/run.sh (default 1) and Docker (default 4). The -# systemd unit always runs 4. +# REST API worker processes. Default 1 under scripts/run.sh, 4 on Docker and +# server installs. # WORKERS= # gRPC @@ -21,7 +21,7 @@ NONCE_TTL_SECONDS=600 # Database DATABASE_DIALECT=sqlite -# Database Encryption (SQLCipher for SQLite, TDE for MySQL, disk-level for Postgres) +# Encrypt the SQLite file with SQLCipher. No effect on MySQL or PostgreSQL. DATABASE_ENCRYPTION_ENABLED=true # 64-char hex (32 bytes). Generate: openssl rand -hex 32 DATABASE_ENCRYPTION_KEY= @@ -31,7 +31,7 @@ DATABASE_FIELD_ENCRYPTION_ENABLED=true # 64-char hex (32 bytes). Generate: openssl rand -hex 32 DATABASE_FIELD_ENCRYPTION_KEY= -# Data Encryption Key (DEK) - used to encrypt all private keys +# Data encryption key: encrypts the server's private keys and keys the sender hash. # 64-char hex (32 bytes). Generate: openssl rand -hex 32 DATA_ENCRYPTION_KEY= @@ -56,7 +56,7 @@ SQLITE_DATABASE_PATH=data/relaysms.db # Broker/backend type: sqlite | redis | rabbitmq CELERY_BROKER_TYPE=sqlite # Worker concurrency. Leave unset to use the per-broker default -# (1 for sqlite, 4 for redis/rabbitmq). +# (1 for sqlite, else 4). # CELERY_WORKER_CONCURRENCY= # sqlite CELERY_BROKER_DB_PATH=data/celery_broker.db @@ -102,7 +102,7 @@ SMTP_IMAP_SERVER= SMTP_IMAP_PORT=993 SMTP_IMAP_USERNAME= SMTP_IMAP_PASSWORD= -# Comma-separated, e.g. INBOX,Spam - catches legit mail misfiled as spam +# Comma-separated, such as INBOX,Spam, to catch payloads misfiled as spam SMTP_IMAP_MAIL_FOLDER=INBOX # Optional mTLS client cert, only if the mail provider requires one SMTP_TLS_CLIENT_CERTIFICATE= @@ -131,7 +131,7 @@ PLATFORMS_ADAPTERS_STATE_DIR=data/platforms/state PLATFORMS_GITHUB_ORGS= # Auth (credentials are managed with ./publisher.sh creds) -# Comma-separated web client origins, e.g. https://console.example.org. Must be +# Comma-separated web client origins, e.g. https://console.example.com. Must be # the same site as the API. Empty if same origin. AUTH_WEB_ORIGINS= # false only for local http development. diff --git a/tests/cli/test_reference.py b/tests/cli/test_reference.py index c712658..9b9ce27 100644 --- a/tests/cli/test_reference.py +++ b/tests/cli/test_reference.py @@ -8,8 +8,8 @@ from publisher.cli import cli -def reference_section(heading: str) -> str: - text = (config.ROOT / "docs" / "reference" / "cli.md").read_text() +def doc_section(page: str, heading: str) -> str: + text = (config.ROOT / "docs" / page).read_text() return text.split(f"## {heading}", 1)[1].split("\n## ", 1)[0] @@ -24,7 +24,7 @@ def commands(group: click.Group, prefix: str = "") -> set[str]: def test_cli_reference_lists_every_publisher_command(): - section = reference_section("publisher.sh") + section = doc_section("reference/cli.md", "publisher.sh") documented = set() for group, cells in re.findall(r"^\| `([a-z-]+)` \| ([^|]+) \|", section, re.M): documented |= {f"{group} {name}" for name in re.findall(r"`([a-z -]+)`", cells)} @@ -39,7 +39,10 @@ def test_cli_reference_lists_every_manage_command(): ) assert usage actual = set(usage[1].split("|")) - documented = set(re.findall(r"`([a-z]+)`", reference_section("manage.sh"))) + section = doc_section("operating/services.md", "manage.sh") + documented = set() + for cell in re.findall(r"^\| (`[^|]+) \|", section, re.M): + documented |= set(re.findall(r"`([a-z]+)[` ]", cell)) - assert actual - documented == set(), "missing from docs/reference/cli.md" - assert documented - actual == set(), "in docs/reference/cli.md, not manage.sh" + assert actual - documented == set(), "missing from docs/operating/services.md" + assert documented - actual == set(), "in docs/operating/services.md, not manage.sh"