From b904758720a01b4a8f0270500d3091f63470ebb5 Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Mon, 2 Nov 2026 16:05:36 +0100 Subject: [PATCH 1/3] feat(stats): accept repeated filter values and add exclude_country_code Signed-off-by: Promise Fru --- docs/openapi.json | 212 ++++++++++++++------------ publisher/api/rest/v1/params.py | 13 ++ publisher/api/rest/v1/stats.py | 29 +++- publisher/models/publication_stats.py | 23 ++- tests/api/rest/test_auth.py | 2 +- tests/api/rest/test_stats.py | 50 ++++++ 6 files changed, 218 insertions(+), 111 deletions(-) diff --git a/docs/openapi.json b/docs/openapi.json index 9c0e32d..05aac51 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1104,80 +1104,90 @@ "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string", - "maxLength": 20, - "pattern": "^[a-zA-Z0-9_-]+$" - }, - { - "type": "null" - } - ], - "description": "Filter by status", + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Filter by any of these statuses. Repeatable.", + "default": [], "title": "Status" }, - "description": "Filter by status" + "description": "Filter by any of these statuses. Repeatable." }, { "name": "platform_name", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string", - "maxLength": 100, - "pattern": "^[a-zA-Z0-9_-]+$" - }, - { - "type": "null" - } - ], - "description": "Filter by platform name", + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Filter by any of these platform names. Repeatable.", + "default": [], "title": "Platform Name" }, - "description": "Filter by platform name" + "description": "Filter by any of these platform names. Repeatable." }, { "name": "protocol", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string", - "maxLength": 20, - "pattern": "^[a-zA-Z0-9_-]+$" - }, - { - "type": "null" - } - ], - "description": "Filter by ingestion protocol", + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Filter by any of these ingestion protocols. Repeatable.", + "default": [], "title": "Protocol" }, - "description": "Filter by ingestion protocol" + "description": "Filter by any of these ingestion protocols. Repeatable." }, { "name": "country_code", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string", - "maxLength": 10, - "pattern": "^[a-zA-Z0-9_-]+$" - }, - { - "type": "null" - } - ], - "description": "Filter by ISO country code", + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Filter by any of these ISO country codes. Repeatable.", + "default": [], "title": "Country Code" }, - "description": "Filter by ISO country code" + "description": "Filter by any of these ISO country codes. Repeatable." + }, + { + "name": "exclude_country_code", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Leave out these ISO country codes. Rows with no country are kept. Repeatable.", + "default": [], + "title": "Exclude Country Code" + }, + "description": "Leave out these ISO country codes. Rows with no country are kept. Repeatable." }, { "name": "since", @@ -1333,80 +1343,90 @@ "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string", - "maxLength": 20, - "pattern": "^[a-zA-Z0-9_-]+$" - }, - { - "type": "null" - } - ], - "description": "Filter by status", + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Filter by any of these statuses. Repeatable.", + "default": [], "title": "Status" }, - "description": "Filter by status" + "description": "Filter by any of these statuses. Repeatable." }, { "name": "platform_name", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string", - "maxLength": 100, - "pattern": "^[a-zA-Z0-9_-]+$" - }, - { - "type": "null" - } - ], - "description": "Filter by platform name", + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Filter by any of these platform names. Repeatable.", + "default": [], "title": "Platform Name" }, - "description": "Filter by platform name" + "description": "Filter by any of these platform names. Repeatable." }, { "name": "protocol", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string", - "maxLength": 20, - "pattern": "^[a-zA-Z0-9_-]+$" - }, - { - "type": "null" - } - ], - "description": "Filter by ingestion protocol", + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Filter by any of these ingestion protocols. Repeatable.", + "default": [], "title": "Protocol" }, - "description": "Filter by ingestion protocol" + "description": "Filter by any of these ingestion protocols. Repeatable." }, { "name": "country_code", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string", - "maxLength": 10, - "pattern": "^[a-zA-Z0-9_-]+$" - }, - { - "type": "null" - } - ], - "description": "Filter by ISO country code", + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Filter by any of these ISO country codes. Repeatable.", + "default": [], "title": "Country Code" }, - "description": "Filter by ISO country code" + "description": "Filter by any of these ISO country codes. Repeatable." + }, + { + "name": "exclude_country_code", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string", + "maxLength": 100, + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "maxItems": 50, + "description": "Leave out these ISO country codes. Rows with no country are kept. Repeatable.", + "default": [], + "title": "Exclude Country Code" + }, + "description": "Leave out these ISO country codes. Rows with no country are kept. Repeatable." }, { "name": "since", diff --git a/publisher/api/rest/v1/params.py b/publisher/api/rest/v1/params.py index 992c226..66f3cd4 100644 --- a/publisher/api/rest/v1/params.py +++ b/publisher/api/rest/v1/params.py @@ -2,13 +2,18 @@ import datetime import hashlib +from typing import Annotated from fastapi import HTTPException, Query, Request +from pydantic import StringConstraints from publisher.api.rest.v1.errors import ApiError from publisher.db.types import as_utc NAME_PATTERN = r"^[a-zA-Z0-9_-]+$" +MAX_FILTER_VALUES = 50 + +FilterValue = Annotated[str, StringConstraints(max_length=100, pattern=NAME_PATTERN)] def filter_query(max_length: int, description: str): @@ -20,6 +25,14 @@ def filter_query(max_length: int, description: str): ) +def filter_list_query(description: str): + return Query( + (), + max_length=MAX_FILTER_VALUES, + description=f"{description}. Repeatable.", + ) + + def page_link(request: Request, cursor: str | None) -> str | None: return str(request.url.include_query_params(cursor=cursor)) if cursor else None diff --git a/publisher/api/rest/v1/stats.py b/publisher/api/rest/v1/stats.py index b7c5cba..f9f6ac9 100644 --- a/publisher/api/rest/v1/stats.py +++ b/publisher/api/rest/v1/stats.py @@ -15,7 +15,12 @@ from publisher.api.rest.v1.auth import AuthContext, authorize, require_scopes from publisher.api.rest.v1.errors import PAGE_QUERY_ERRORS, error_responses -from publisher.api.rest.v1.params import check_time_range, filter_query, page_link +from publisher.api.rest.v1.params import ( + FilterValue, + check_time_range, + filter_list_query, + page_link, +) from publisher.api.rest.v1.schemas import ( PublicationStatsPage, PublicationStatsSummary, @@ -34,10 +39,21 @@ def stats_filters( - status: str | None = filter_query(20, "Filter by status"), - platform_name: str | None = filter_query(100, "Filter by platform name"), - protocol: str | None = filter_query(20, "Filter by ingestion protocol"), - country_code: str | None = filter_query(10, "Filter by ISO country code"), + status: tuple[FilterValue, ...] = filter_list_query( + "Filter by any of these statuses" + ), + platform_name: tuple[FilterValue, ...] = filter_list_query( + "Filter by any of these platform names" + ), + protocol: tuple[FilterValue, ...] = filter_list_query( + "Filter by any of these ingestion protocols" + ), + country_code: tuple[FilterValue, ...] = filter_list_query( + "Filter by any of these ISO country codes" + ), + exclude_country_code: tuple[FilterValue, ...] = filter_list_query( + "Leave out these ISO country codes. Rows with no country are kept" + ), since: datetime.datetime | None = Query( None, description="Created at or after (ISO-8601, UTC if no offset)", @@ -52,7 +68,8 @@ def stats_filters( status=status, platform_name=platform_name, protocol=protocol, - country_code=country_code, + country_code=tuple(code.upper() for code in country_code), + exclude_country_code=tuple(code.upper() for code in exclude_country_code), since=since, until=until, ) diff --git a/publisher/models/publication_stats.py b/publisher/models/publication_stats.py index fd14793..bd66771 100644 --- a/publisher/models/publication_stats.py +++ b/publisher/models/publication_stats.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any -from sqlalchemy import Index, String, func, select +from sqlalchemy import Index, String, func, or_, select from sqlalchemy.orm import Mapped, Session, mapped_column from publisher.db import Base, pagination @@ -89,10 +89,11 @@ def record( @dataclass(frozen=True) class StatsFilters: - status: str | None = None - platform_name: str | None = None - protocol: str | None = None - country_code: str | None = None + status: tuple[str, ...] = () + platform_name: tuple[str, ...] = () + protocol: tuple[str, ...] = () + country_code: tuple[str, ...] = () + exclude_country_code: tuple[str, ...] = () since: datetime.datetime | None = None until: datetime.datetime | None = None @@ -100,9 +101,15 @@ class StatsFilters: def _filter_clauses(filters: StatsFilters) -> list: clauses = [] for name in ("status", "platform_name", "protocol", "country_code"): - value = getattr(filters, name) - if value is not None: - clauses.append(getattr(PublicationStats, name) == value) + values = getattr(filters, name) + if values: + clauses.append(getattr(PublicationStats, name).in_(values)) + if filters.exclude_country_code: + country = PublicationStats.country_code + # Keep rows with no country, which NOT IN never matches. + clauses.append( + or_(country.not_in(filters.exclude_country_code), country.is_(None)) + ) if filters.since is not None: clauses.append(PublicationStats.created_at >= filters.since) if filters.until is not None: diff --git a/tests/api/rest/test_auth.py b/tests/api/rest/test_auth.py index 47ac102..2029cc2 100644 --- a/tests/api/rest/test_auth.py +++ b/tests/api/rest/test_auth.py @@ -228,7 +228,7 @@ def test_cleanup_deletes_only_expired_sessions(client, password): [ ( "/v1/stats/publications?status=bad%0Aline", - "query.status: String should match pattern '^[a-zA-Z0-9_-]+$', " + "query.status.0: String should match pattern '^[a-zA-Z0-9_-]+$', " "got 'bad\\nline'", ), ( diff --git a/tests/api/rest/test_stats.py b/tests/api/rest/test_stats.py index 9303085..3afdedc 100644 --- a/tests/api/rest/test_stats.py +++ b/tests/api/rest/test_stats.py @@ -230,6 +230,56 @@ def test_filters_and_time_range(authed_client, seeded): assert {item["id"] for item in response.json()["data"]} == expected +def test_repeated_filters_match_any_value(authed_client, seeded): + extra = [dict(seeded[1], status="rejected"), dict(seeded[1], country_code="GH")] + _seed(extra) + rows = seeded + extra + + response = authed_client.get( + "/v1/stats/publications", + params={"status": ["failed", "published"], "country_code": ["cm", "GH"]}, + ) + + assert response.status_code == 200 + expected = { + n + for n, row in enumerate(rows, start=1) + if row["status"] in ("failed", "published") + and row["country_code"] in ("CM", "GH") + } + assert {item["id"] for item in response.json()["data"]} == expected + + +def test_exclude_country_code_keeps_rows_without_a_country(authed_client): + _seed( + dict( + platform_name="gmail", + protocol="sms", + status="published", + country_code=country, + created_at=BASE_TIME, + ) + for country in ("CM", "NG", "GH", None) + ) + + response = authed_client.get( + "/v1/stats/publications", params={"exclude_country_code": ["ng", "GH"]} + ) + + assert response.status_code == 200 + rows = response.json()["data"] + assert {row["country_code"] for row in rows} == {"CM", None} + + +@pytest.mark.parametrize("count, status", [(50, 200), (51, 422)]) +def test_filter_values_are_capped_at_50(authed_client, count, status): + response = authed_client.get( + "/v1/stats/publications", params={"country_code": ["CM"] * count} + ) + + assert response.status_code == status + + @pytest.mark.parametrize( "params, status", [ From 06f860ee40630387c62a41433eee085f45b5fbdf Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Mon, 2 Nov 2026 20:33:53 +0100 Subject: [PATCH 2/3] ci: label fork PRs that don't follow CONTRIBUTING.md and close them after 5 days without activity Signed-off-by: Promise Fru --- .github/workflows/pr-guidelines.yml | 129 ++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 .github/workflows/pr-guidelines.yml diff --git a/.github/workflows/pr-guidelines.yml b/.github/workflows/pr-guidelines.yml new file mode 100644 index 0000000..96b8f92 --- /dev/null +++ b/.github/workflows/pr-guidelines.yml @@ -0,0 +1,129 @@ +name: PR guidelines + +# Labels and comments on fork PRs that don't follow CONTRIBUTING.md, and closes +# them after 5 days without activity. It runs with a write token: never check +# out or run the PR's code. +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + schedule: + - cron: "0 6 * * *" + workflow_dispatch: + inputs: + pr: + description: Fork PR number to check. Leave empty to check every open fork PR. + required: false + +permissions: {} + +env: + LABEL: not-following-guidelines + STALE_LABEL: stale + +jobs: + check: + if: github.event_name != 'schedule' + runs-on: ubuntu-latest + concurrency: + group: pr-guidelines-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/github-script@v9 + env: + PR_NUMBER: ${{ inputs.pr }} + with: + script: | + const { LABEL, STALE_LABEL, PR_NUMBER } = process.env; + const MARKER = ""; + const { owner, repo } = context.repo; + + let prs; + if (context.payload.pull_request) { + prs = [context.payload.pull_request]; + } else if (PR_NUMBER) { + prs = [(await github.rest.pulls.get({ owner, repo, pull_number: Number(PR_NUMBER) })).data]; + } else { + prs = await github.paginate(github.rest.pulls.list, { owner, repo, state: "open" }); + } + // Checks only open fork PRs, including ones whose fork was deleted (head.repo is null). + prs = prs.filter((pr) => pr.state === "open" && pr.head.repo?.full_name !== `${owner}/${repo}`); + + const { html_url, default_branch } = context.payload.repository; + const GUIDE = `${html_url}/blob/${default_branch}/CONTRIBUTING.md`; + + for (const pr of prs) { + const issue_number = pr.number; + const files = await github.paginate(github.rest.pulls.listFiles, { + owner, repo, pull_number: issue_number, + }); + + const problems = []; + if (pr.base.ref !== "staging") { + problems.push(`The PR targets \`${pr.base.ref}\`. Every PR [targets \`staging\`](${GUIDE}#branches): use **Edit** next to the title to change the base branch.`); + } + const paths = files.map((f) => f.filename); + const changesCode = paths.some((p) => /^(publisher|scripts|sdk\/src)\/|^[^/]+\.sh$/.test(p)); + const changesTests = paths.some((p) => /^(tests|sdk\/tests)\//.test(p)); + if (changesCode && !changesTests) { + problems.push(`The PR changes code but no tests. [New behavior and bug fixes need tests](${GUIDE}#pull-request-checklist).`); + } + + const labels = pr.labels.map((l) => l.name); + let body; + if (problems.length) { + body = [ + MARKER, + `Thanks for the PR. It doesn't follow the [contribution guide](${GUIDE}) yet:`, + "", + ...problems.map((p) => `- ${p}`), + "", + `Also [run the checks CI runs](${GUIDE}#run-the-checks-ci-runs) before you push. This comment updates when you push or edit the PR. Without activity, the PR is closed after 5 days.`, + ].join("\n"); + if (!labels.includes(LABEL)) { + await github.rest.issues.createLabel({ owner, repo, name: LABEL, color: "d93f0b", + description: "The PR doesn't follow CONTRIBUTING.md yet" }).catch(() => {}); + await github.rest.issues.addLabels({ owner, repo, issue_number, labels: [LABEL] }); + } + } else { + body = `${MARKER}\nThanks, the base branch and tests now follow the [contribution guide](${GUIDE}).`; + for (const name of [LABEL, STALE_LABEL].filter((n) => labels.includes(n))) { + await github.rest.issues.removeLabel({ owner, repo, issue_number, name }); + } + } + + const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number }); + const existing = comments.find((c) => c.user.type === "Bot" && c.body.includes(MARKER)); + if (existing) { + if (existing.body !== body) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } + } else if (problems.length) { + await github.rest.issues.createComment({ owner, repo, issue_number, body }); + } + } + + close-stale: + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + permissions: + actions: write + issues: write + pull-requests: write + steps: + - uses: actions/stale@v11 + with: + only-pr-labels: ${{ env.LABEL }} + stale-pr-label: ${{ env.STALE_LABEL }} + days-before-issue-stale: -1 + days-before-issue-close: -1 + days-before-pr-stale: 3 + days-before-pr-close: 2 + stale-pr-message: >- + This PR has had no activity for 3 days and still doesn't follow the + contribution guide. It will be closed in 2 days without new activity. + close-pr-message: >- + Closed because it didn't follow the contribution guide and had no + activity for 5 days. Push the fixes and open a new PR when it's ready. From da81c91d732dfbe3436c59cb7feb5c1c448edb3e Mon Sep 17 00:00:00 2001 From: Promise Fru Date: Mon, 2 Nov 2026 20:46:06 +0100 Subject: [PATCH 3/3] style(ci): drop the write-token note from the PR guidelines header Signed-off-by: Promise Fru --- .github/workflows/pr-guidelines.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr-guidelines.yml b/.github/workflows/pr-guidelines.yml index 96b8f92..b168a39 100644 --- a/.github/workflows/pr-guidelines.yml +++ b/.github/workflows/pr-guidelines.yml @@ -1,8 +1,7 @@ name: PR guidelines # Labels and comments on fork PRs that don't follow CONTRIBUTING.md, and closes -# them after 5 days without activity. It runs with a write token: never check -# out or run the PR's code. +# them after 5 days without activity. on: pull_request_target: types: [opened, edited, reopened, synchronize]