From 99c35fb929b3ef651614eefcd1eb76b0fc15a713 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Mon, 5 Oct 2026 15:31:19 -0700 Subject: [PATCH 01/11] Simplify switching between installed AWS CLI v2 versions (#8361) Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 71e627446f..14c3c23f82 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,7 @@ generic with minimal need for project-specific behavior. - [Terraform](#212-terraform), to manage deployments -- [AWS CLI v2], for programmatic invocations to AWS services. Since v2 is not - available on PyPI, it must be installed separately. Install the version pinned - by Azul defined in a variable called `azul_awscli_version` in - [environment.py]. Follow the [AWS instructions for installing past - releases][AWS CLI v2]. +- [AWS CLI v2](#213-aws-cli-v2), for command line interactions with AWS services - Optionally, the [GitHub CLI]. You should install the version set at `azul_ghcli_version` in [environment.py], @@ -184,6 +180,45 @@ long-term use without the cache. [plugin cache]: https://developer.hashicorp.com/terraform/cli/config/config-file#provider-plugin-cache +### 2.1.3 AWS CLI v2 + +Azul requires a specific version of the AWS CLI v2, which is defined in a +variable called `azul_awscli_version` in [environment.py]. Since v2 is not +available on PyPI, it must be installed separately. Follow the official [AWS +instructions for installing past releases][AWS CLI v2]. + +The official instructions install the CLI system-wide, and each installation +replaces the previously installed version. That is inconvenient because the +pinned version is updated routinely, as part of the biweekly upgrade, and the +stable deployments typically trail the lower ones by a week or two. Operators +therefore have to switch back and forth between versions of the CLI as they +switch between working copies. + +[awsclienv] avoids that but it currently only supports macOS. It installs any +number of versions of the AWS CLI v2 side by side, underneath your home +directory, allowing you to switch between them via the `AWSCLIENV_VERSION` +environment variable. Install it as described in the [Installation +section][awsclienv install] of the `awsclienv` README. Then add + +```python +'AWSCLIENV_VERSION': '{azul_awscli_version}', +``` + +to your `environment.local.py` file at the root of the working copy. Run +`. environment` or `_refresh`. Then run + +``` +awsclienv install +``` + +Verify with `make check_awscli` that Azul is happy. If that target fails at any +point in the future, re-run `awsclienv install` to install the version required +by Azul. + +[awsclienv]: https://github.com/hannes-ucsc/awsclienv +[awsclienv install]: https://github.com/hannes-ucsc/awsclienv#installation + + ## 2.2 Runtime Prerequisites (Infrastructure) An instance of the HCA [Data Store] aka DSS. The URL of that instance can be From ff2fd484c9acf2d91b72e4678e2f43897e8ca709 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Mon, 5 Oct 2026 18:01:18 -0700 Subject: [PATCH 02/11] Fix: Controllers not supporting APATs don't reject them early (#8367) Co-Authored-By: Claude Opus 5 (1M context) --- src/azul/chalice.py | 14 ++++++++++++++ src/azul/service/repository_controller.py | 1 + src/azul/service/user_controller.py | 1 + test/service/test_user_controller.py | 12 ++++++++++++ 4 files changed, 28 insertions(+) diff --git a/src/azul/chalice.py b/src/azul/chalice.py index 342946b2ad..96bed7d098 100644 --- a/src/azul/chalice.py +++ b/src/azul/chalice.py @@ -18,6 +18,7 @@ from typing import ( Any, Callable, + ClassVar, Iterator, Literal, Mapping, @@ -39,6 +40,7 @@ NotFoundError, Request, Response, + UnauthorizedError, ) import chevron from furl import ( @@ -50,6 +52,7 @@ ) from azul.auth import ( Authentication, + PersonalAccessTokenAuthentication, ) from azul.csp import ( CSP, @@ -974,6 +977,11 @@ def default_specs(self): class Controller: app: AzulChaliceApp + #: Whether the endpoints handled by this controller accept a personal + #: access token (APAT) as the credentials of the requesting user. + #: + accepts_personal_access_token: ClassVar[bool] = False + @property def lambda_context(self) -> LambdaContext: return not_none(self.app.lambda_context) @@ -986,6 +994,12 @@ def _authentication(self, request: Request) -> Authentication | None: authentication = getattr(request, 'authentication', None) if authentication is not None: assert isinstance(authentication, Authentication) + if ( + isinstance(authentication, PersonalAccessTokenAuthentication) + and not self.accepts_personal_access_token + ): + raise UnauthorizedError('This endpoint does not support a ' + 'personal access token (APAT)') return authentication def _query_params(self, request: Request) -> MultiDict: diff --git a/src/azul/service/repository_controller.py b/src/azul/service/repository_controller.py index cab95363f7..4ee7dea275 100644 --- a/src/azul/service/repository_controller.py +++ b/src/azul/service/repository_controller.py @@ -82,6 +82,7 @@ class RepositoryController(ServiceController): + accepts_personal_access_token = True @cached_property def _repository_service(self) -> RepositoryService: diff --git a/src/azul/service/user_controller.py b/src/azul/service/user_controller.py index 6baddedfdb..c79db4d692 100644 --- a/src/azul/service/user_controller.py +++ b/src/azul/service/user_controller.py @@ -60,6 +60,7 @@ class UserController(Controller): + accepts_personal_access_token = True _json_content_type = 'application/json' _form_content_type = 'application/x-www-form-urlencoded' diff --git a/test/service/test_user_controller.py b/test/service/test_user_controller.py index 07fb229e0e..a70215344e 100644 --- a/test/service/test_user_controller.py +++ b/test/service/test_user_controller.py @@ -432,3 +432,15 @@ def test_revoke_rejects_pat(self): def test_revoke_unauthenticated(self): response = self._revoke() self.assertEqual(401, response.status) + + def test_unsupported_endpoint_rejects_pat(self): + apat = self._authorize_and_mint() + url = str(self.base_url.set(path='/index/summary')) + headers = {'Authorization': f'Bearer {apat.token}'} + response = self._http_client.request('GET', url, headers=headers) + self.assertEqual(401, response.status) + self.assertEqual({ + 'Code': 'UnauthorizedError', + 'Message': 'This endpoint does not support a ' + 'personal access token (APAT)' + }, json.loads(response.data)) From 4c37d68b02385cfb8759164bfeece8fa88dc1bce Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Mon, 5 Oct 2026 22:29:00 -0700 Subject: [PATCH 03/11] Fix: FedRAMP inventory lists each OpenSearch domain twice (DataBiosphere/azul-private#431) Co-Authored-By: Claude Opus 5 (1M context) --- .../compliance/fedramp_inventory_service.py | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/azul/compliance/fedramp_inventory_service.py b/src/azul/compliance/fedramp_inventory_service.py index 63c0170880..69f3855ad0 100644 --- a/src/azul/compliance/fedramp_inventory_service.py +++ b/src/azul/compliance/fedramp_inventory_service.py @@ -46,6 +46,7 @@ aws, ) from azul.lib import ( + R, cached_property, ) from azul.lib.types import ( @@ -123,6 +124,16 @@ class Mapper(metaclass=ABCMeta): def map(self, resource: ResourceConfig) -> Iterable[InventoryRow]: raise NotImplementedError + def validate_types(self, resource_types: Sequence[str]) -> None: + """ + Verify this mapper's assumptions about the types of resource that are + going to be discovered. Called once, before discovery. + """ + undiscovered = self._supported_resource_types() - set(resource_types) + assert not undiscovered, R( + 'Mapper supports resource types that are not discovered', + type(self).__name__, sorted(undiscovered)) + def _common_fields(self, resource: ResourceConfig, *, @@ -163,19 +174,31 @@ def map(self, resource: ResourceConfig) -> Iterator[InventoryRow]: ) -class ElasticSearchMapper(Mapper): +class OpenSearchMapper(Mapper): + #: AWS Config reports every OpenSearch domain under this deprecated type + #: as well as under the modern one this mapper supports. Discovering both + #: would inventory each domain twice, so only the modern one is + #: discovered, and `validate_types` ensures that remains so. + #: + _deprecated_type = 'AWS::Elasticsearch::Domain' def _supported_resource_types(self) -> set[str]: - return {'AWS::Elasticsearch::Domain'} + return {'AWS::OpenSearch::Domain'} + + def validate_types(self, resource_types: Sequence[str]) -> None: + super().validate_types(resource_types) + assert self._deprecated_type not in resource_types, R( + 'Discovering this type would inventory every OpenSearch domain ' + 'twice', self._deprecated_type) def map(self, resource: ResourceConfig) -> Iterator[InventoryRow]: yield InventoryRow( asset_type='AWS OpenSearch Domain', - baseline_config=resource.config['elasticsearchVersion'], + baseline_config=resource.config['EngineVersion'], is_public=YesNo.no, is_virtual=YesNo.yes, - network_id=resource.config['endpoints'].get('vpc'), - patch_level=resource.config.get('serviceSoftwareOptions', {}).get('currentVersion'), + network_id=resource.config['DomainEndpoints'].get('vpc'), + patch_level=resource.config.get('ServiceSoftwareOptions', {}).get('CurrentVersion'), software_product_name='AWS OpenSearch', **self._common_fields(resource) ) @@ -520,6 +543,8 @@ def get_linenno(o: type) -> int: def resource_ids_by_type(self) -> defaultdict[str, list[str]]: resource_ids_by_type = defaultdict(list) + for mapper in self._mappers: + mapper.validate_types(self._all_aws_resource_types) for resource_type in self._all_aws_resource_types: args = dict(resourceType=resource_type) while True: @@ -813,7 +838,6 @@ def _wiki_content(self, resources: Iterable[ResourceConfig]) -> str: 'AWS::ElasticLoadBalancing::LoadBalancer', 'AWS::ElasticLoadBalancingV2::Listener', 'AWS::ElasticLoadBalancingV2::LoadBalancer', - 'AWS::Elasticsearch::Domain', 'AWS::EventSchemas::Discoverer', 'AWS::EventSchemas::Registry', 'AWS::EventSchemas::RegistryPolicy', From e146e7070b7ac5da921dadd7b677ced7f57452da Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Wed, 7 Oct 2026 20:50:12 -0700 Subject: [PATCH 04/11] Fix: GitHub Actions deprecated Node20 (#8381) Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1042297809..3a03c16318 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,11 +12,11 @@ jobs: timeout-minutes: 45 steps: - - uses: 'actions/checkout@v4' + - uses: 'actions/checkout@v7' - run: 'cat environment.boot >> "$GITHUB_ENV"' - - uses: 'actions/setup-python@v5' + - uses: 'actions/setup-python@v7' with: python-version: '${{ env.azul_python_version }}' @@ -82,7 +82,7 @@ jobs: make check_clean coverage xml - - uses: 'actions/upload-artifact@v4' + - uses: 'actions/upload-artifact@v7' with: name: 'coverage-file' path: 'coverage.xml' @@ -92,11 +92,11 @@ jobs: runs-on: 'ubuntu-22.04' steps: - - uses: 'actions/checkout@v4' + - uses: 'actions/checkout@v7' - run: 'cat environment.boot >> "$GITHUB_ENV"' - - uses: 'actions/download-artifact@v4' + - uses: 'actions/download-artifact@v8' with: name: 'coverage-file' @@ -110,11 +110,11 @@ jobs: needs: 'test' runs-on: 'ubuntu-22.04' steps: - - uses: 'actions/checkout@v4' + - uses: 'actions/checkout@v7' - run: 'cat environment.boot >> "$GITHUB_ENV"' - - uses: 'actions/download-artifact@v4' + - uses: 'actions/download-artifact@v8' with: name: 'coverage-file' @@ -134,16 +134,16 @@ jobs: language: [ 'python', 'javascript' ] steps: - - uses: 'actions/checkout@v4' + - uses: 'actions/checkout@v7' - run: 'cat environment.boot >> "$GITHUB_ENV"' - - uses: 'github/codeql-action/init@v3' + - uses: 'github/codeql-action/init@v4' with: languages: '${{ matrix.language }}' config-file: './.github/codeql/codeql-config.yml' queries: 'security-and-quality' - - uses: 'github/codeql-action/analyze@v3' + - uses: 'github/codeql-action/analyze@v4' with: category: '/language:${{matrix.language}}' From 0f54da2202c5e83847501d8274a2ec7892bbdd77 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Wed, 7 Oct 2026 22:30:47 -0700 Subject: [PATCH 05/11] [1/2] Pin GitHub actions to commits A version tag, even a fully specified one, can be moved by the project that publishes the action, so a commit is the only reference that is fixed. Add a script that rewrites every pin in the workflows to the commit behind the most recent fully specified version tag, and a checklist item that runs it as part of the biweekly upgrade. The next commit is the result of running it. Co-Authored-By: Claude Opus 5 (1M context) --- .github/ISSUE_TEMPLATE/upgrade.md | 1 + .github/workflows/ci.yml | 3 - .mypy.ini | 1 + scripts/upgrade_actions.py | 167 ++++++++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 scripts/upgrade_actions.py diff --git a/.github/ISSUE_TEMPLATE/upgrade.md b/.github/ISSUE_TEMPLATE/upgrade.md index 3773c8aaf7..dfa920e85b 100644 --- a/.github/ISSUE_TEMPLATE/upgrade.md +++ b/.github/ISSUE_TEMPLATE/upgrade.md @@ -56,6 +56,7 @@ _period: 14 days - [ ] … update to [AWS CLI v2](https://github.com/aws/aws-cli/blob/v2/CHANGELOG.rst) or no update available - [ ] … update to [GitHub CLI](https://github.com/cli/cli/releases), having run `make gh_checksums` afterwards or no update available - [ ] … update to [uv](https://github.com/astral-sh/uv/releases), following the instructions next to the pin in `pyproject.toml` or no update available + - [ ] … update the [GitHub action pins](https://github.com/DataBiosphere/azul/blob/develop/.github/workflows/ci.yml) by running `python scripts/upgrade_actions.py` and reviewing the changes it links to or no update available - [ ] Created issues for any deferred updates to … - [ ] … the next major or minor Python version or such an issue already exists - [ ] … the next major Docker version or such an issue already exists diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a03c16318..2826dfd061 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,6 @@ jobs: with: python-version: '${{ env.azul_python_version }}' - # The action is referenced by commit, not by tag, because the project - # doesn't publish a tag for the major version this commit belongs to. - # - uses: 'astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d' # v10.0.1 # The action reads `required-version` from `pyproject.toml`. with: diff --git a/.mypy.ini b/.mypy.ini index ff3994fa29..ca39da67de 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -81,6 +81,7 @@ modules = scripts.check_transitive_requirements, scripts.export_environment, scripts.tf_clean_providers, + scripts.upgrade_actions, packages = diff --git a/scripts/upgrade_actions.py b/scripts/upgrade_actions.py new file mode 100644 index 0000000000..5235677378 --- /dev/null +++ b/scripts/upgrade_actions.py @@ -0,0 +1,167 @@ +""" +Update the version pins of the GitHub actions used in this project's workflows. + +Every action is pinned to a commit SHA, followed by a comment naming the +version that SHA belongs to. A SHA is the only reference that upstream cannot +repoint, which a version tag otherwise is free to be, even a fully specified +one. + +For each action, the script determines the most recent release whose tag is a +fully specified version, resolves that tag to a SHA and rewrites the pin. It +leaves the modified workflows in the working copy for review; it does not +commit them. Actions pinned to a version the script can't determine are left +alone and reported. +""" +import argparse +import json +import logging +from pathlib import ( + Path, +) +import re +import subprocess +import sys +from typing import ( + Any, +) + +import attrs + +from azul.lib import ( + R, +) +from azul.logging import ( + configure_script_logging, +) + +log = logging.getLogger(__name__) + +_project_root = Path(__file__).resolve().parent.parent +_workflow_dir = _project_root / '.github' / 'workflows' + +#: Matches a step's reference to an action, capturing the owner and name of the +#: repository defining it, the path to the action within that repository, the +#: pinned reference and the comment naming the pinned version, if any +#: +_use = re.compile( + r"""(?Puses:\s*'?) + (?P[\w.-]+/[\w.-]+) + (?P(?:/[\w.-]+)*) + @(?P[\w.-]+) + (?P'?) + (?:[^\S\n]*\#[^\S\n]*(?P\S+))? + """, + re.VERBOSE) + +#: Matches a tag naming a fully specified version, as opposed to one that only +#: names a major or minor version, or a pre-release +#: +_version = re.compile(r'v\d+\.\d+\.\d+') + + +@attrs.frozen(kw_only=True) +class Action: + repo: str + version: str + sha: str + + @property + def url(self) -> str: + return f'https://github.com/{self.repo}' + + def compare_url(self, other: str) -> str: + return f'{self.url}/compare/{other}...{self.version}' + + +def main(argv): + parser = argparse.ArgumentParser(description=__doc__) + parser.parse_args(argv) + workflows = sorted(_workflow_dir.glob('*.yml')) + assert workflows, R('No workflows found', str(_workflow_dir)) + actions: dict[str, Action | None] = {} + for workflow in workflows: + content = workflow.read_text() + updated = _update(content, actions) + if updated == content: + log.info('%s is up to date', _relative(workflow)) + else: + workflow.write_text(updated) + log.info('%s was updated, review it with `git diff`', + _relative(workflow)) + + +def _relative(path: Path) -> str: + return str(path.relative_to(_project_root)) + + +def _update(content: str, actions: dict[str, Action | None]) -> str: + def replace(match: re.Match) -> str: + repo = match.group('repo') + if repo not in actions: + actions[repo] = _latest_action(repo) + action = actions[repo] + # The comment names the pinned version more precisely than the pinned + # reference does, once that reference is a SHA + current = match.group('version') or match.group('ref') + if action is None: + log.warning('%s is pinned to %s and was left alone', repo, current) + return match.group(0) + elif action.version == current: + log.info('%s is current at %s', repo, current) + else: + log.info('%s goes from %s to %s, see %s', + repo, current, action.version, + action.compare_url(current)) + return ''.join([ + match.group('prefix'), + repo, + match.group('path'), + '@', action.sha, + match.group('suffix'), + ' # ', action.version + ]) + + return _use.sub(replace, content) + + +def _latest_action(repo: str) -> Action | None: + tags = _tags(repo) + versions = sorted( + (tuple(map(int, tag[1:].split('.'))), tag) + for tag in tags + if _version.fullmatch(tag) + ) + if versions: + _, version = versions[-1] + return Action(repo=repo, version=version, sha=_sha(repo, tags[version])) + else: + return None + + +def _tags(repo: str) -> dict[str, Any]: + refs = _gh(f'repos/{repo}/git/matching-refs/tags/v') + return { + ref['ref'].removeprefix('refs/tags/'): ref['object'] + for ref in refs + } + + +def _sha(repo: str, object: Any) -> str: + # An annotated tag points at a tag object, which in turn points at the + # commit. A lightweight one points at the commit directly. + if object['type'] == 'tag': + object = _gh(object['url'])['object'] + sha = object['sha'] + assert re.fullmatch(r'[0-9a-f]{40}', sha), R('Not a commit SHA', repo, sha) + return sha + + +def _gh(endpoint: str) -> Any: + result = subprocess.run(['gh', 'api', endpoint], + capture_output=True, text=True, check=True) + return json.loads(result.stdout) + + +if __name__ == '__main__': + configure_script_logging(log) + main(sys.argv[1:]) From 3db9e5bc482956ad58d4469f66ede37fb4641b93 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Wed, 7 Oct 2026 22:33:13 -0700 Subject: [PATCH 06/11] [2/2] Pin GitHub actions to commits The result of running `python scripts/upgrade_actions.py`. Besides replacing each tag with the commit behind it, this updates `setup-uv` from v10.0.1 to v10.2.0. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2826dfd061..bce793ab74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,15 +12,15 @@ jobs: timeout-minutes: 45 steps: - - uses: 'actions/checkout@v7' + - uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # v7.0.1 - run: 'cat environment.boot >> "$GITHUB_ENV"' - - uses: 'actions/setup-python@v7' + - uses: 'actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97' # v7.0.0 with: python-version: '${{ env.azul_python_version }}' - - uses: 'astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d' # v10.0.1 + - uses: 'astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7' # v10.2.0 # The action reads `required-version` from `pyproject.toml`. with: enable-cache: true @@ -79,7 +79,7 @@ jobs: make check_clean coverage xml - - uses: 'actions/upload-artifact@v7' + - uses: 'actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9' # v7.0.2 with: name: 'coverage-file' path: 'coverage.xml' @@ -89,15 +89,15 @@ jobs: runs-on: 'ubuntu-22.04' steps: - - uses: 'actions/checkout@v7' + - uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # v7.0.1 - run: 'cat environment.boot >> "$GITHUB_ENV"' - - uses: 'actions/download-artifact@v8' + - uses: 'actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333' # v8.0.2 with: name: 'coverage-file' - - uses: 'codecov/codecov-action@v7' + - uses: 'codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5' # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} files: 'coverage.xml' @@ -107,15 +107,15 @@ jobs: needs: 'test' runs-on: 'ubuntu-22.04' steps: - - uses: 'actions/checkout@v7' + - uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # v7.0.1 - run: 'cat environment.boot >> "$GITHUB_ENV"' - - uses: 'actions/download-artifact@v8' + - uses: 'actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333' # v8.0.2 with: name: 'coverage-file' - - uses: 'coverallsapp/github-action@v2' + - uses: 'coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329' # v2.3.8 with: github-token: ${{ secrets.COVERALLS_REPO_TOKEN }} @@ -131,16 +131,16 @@ jobs: language: [ 'python', 'javascript' ] steps: - - uses: 'actions/checkout@v7' + - uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' # v7.0.1 - run: 'cat environment.boot >> "$GITHUB_ENV"' - - uses: 'github/codeql-action/init@v4' + - uses: 'github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2' # v4.38.2 with: languages: '${{ matrix.language }}' config-file: './.github/codeql/codeql-config.yml' queries: 'security-and-quality' - - uses: 'github/codeql-action/analyze@v4' + - uses: 'github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2' # v4.38.2 with: category: '/language:${{matrix.language}}' From b9c46e9c27a9ff44587dddfa2aa9830c7609dfec Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Wed, 7 Oct 2026 23:26:53 -0700 Subject: [PATCH 07/11] Enable Dependabot security updates for GitHub actions Routine version updates remain suppressed, as they are for `uv`; the pins are maintained by `scripts/upgrade_actions.py` during the biweekly upgrade. Co-Authored-By: Claude Opus 5 (1M context) --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 63d0f118e4..d042335e5f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,16 @@ updates: # Ignored if open-pull-requests-limit is 0 schedule: interval: 'weekly' + + # The pins in the workflows are updated as part of the biweekly upgrade, by + # `scripts/upgrade_actions.py`. This entry is only here so that security + # updates are raised against them in between. + # + - package-ecosystem: 'github-actions' + directory: '/' + target-branch: 'develop' + # Disable routine dependency updates, but keep security updates active + open-pull-requests-limit: 0 + # Ignored if open-pull-requests-limit is 0 + schedule: + interval: 'weekly' From f9c983556964ab8a45c39ad64b54db713341122e Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Thu, 8 Oct 2026 10:28:53 -0700 Subject: [PATCH 08/11] Fix: Invalid access token causes 500 response from /user/token and /user/revoke (#8375) Both endpoints extend the defect to tokens issued to another client and to users that never completed the authorization code flow. A 400 from the `tokeninfo` endpoint is a condition we expect and know how to report, so `token_info` raises a dedicated exception for it instead of failing an assertion. A response we aren't prepared to interpret keeps its assertion, and the 5xx it causes, since that mismatch crosses the boundary to Google rather than indicating an inconsistency of our own. Co-Authored-By: Claude Opus 5 (1M context) --- src/azul/oauth2.py | 23 +++++++++--- src/azul/service/user_controller.py | 33 ++++++++++++++++-- src/azul/terra.py | 4 +++ test/service/test_user_controller.py | 52 ++++++++++++++++++++++++++-- 4 files changed, 102 insertions(+), 10 deletions(-) diff --git a/src/azul/oauth2.py b/src/azul/oauth2.py index ffa7e94fdf..8cb752c4fd 100644 --- a/src/azul/oauth2.py +++ b/src/azul/oauth2.py @@ -99,6 +99,13 @@ class TokenInfoResponse(TypedDict): access_type: str # "online" +class InvalidAccessTokenError(Exception): + """ + The authorization server rejected the access token. It may be expired, + revoked or forged. + """ + + class OAuth2Client(HasCachedHttpClient): """ A client for Google's implementation of the OAuth 2.0 authorization server @@ -184,11 +191,14 @@ def token_info(self, access_token: str) -> TokenInfoResponse: url = furl(url='https://www.googleapis.com/oauth2/v3/tokeninfo', args=dict(access_token=access_token)) response = self._http_client.request('GET', str(url)) - assert response.status != 400, R('The token is not valid') - assert response.status == 200, R('Unexpected response status', response.status) - response = json.loads(response.data) - assert is_of_type(response, TokenInfoResponse) - return response + if response.status == 400: + raise InvalidAccessTokenError + else: + assert response.status == 200, R('Unexpected response status', + response.status) + response = json.loads(response.data) + assert is_of_type(response, TokenInfoResponse) + return response TokenCredentials = google.oauth2.credentials.Credentials @@ -277,6 +287,9 @@ def validate(self): token must not be expired and the service account must belong to the current Google Cloud project. + :raise InvalidAccessTokenError: if the authorization server rejects the + token + :raise AssertionError: if the token is definitely invalid :raise Exception: if the validity of the token cannot be determined diff --git a/src/azul/service/user_controller.py b/src/azul/service/user_controller.py index c79db4d692..6a17cd0017 100644 --- a/src/azul/service/user_controller.py +++ b/src/azul/service/user_controller.py @@ -1,3 +1,9 @@ +from collections.abc import ( + Iterator, +) +from contextlib import ( + contextmanager, +) from copy import ( copy, ) @@ -43,6 +49,7 @@ ) from azul.oauth2 import ( Authorization, + InvalidAccessTokenError, ) from azul.openapi.responses import ( json_content, @@ -53,6 +60,8 @@ optional, ) from azul.service.user_service import ( + ForeignTokenException, + UnknownUserException, UserService, ) @@ -320,7 +329,8 @@ def _token(self) -> JSON: if isinstance(auth, PersonalAccessTokenAuthentication): raise BadRequestError('Cannot exchange a personal access token for another') elif isinstance(auth, AccessTokenAuthentication): - apat_auth = self._service.mint_personal_access_token(auth) + with self._rejecting_unusable_token(auth): + apat_auth = self._service.mint_personal_access_token(auth) return {'token': apat_auth.token} else: raise UnauthorizedError('Valid access token required') @@ -330,7 +340,26 @@ def _revoke(self) -> Response: if isinstance(auth, PersonalAccessTokenAuthentication): raise BadRequestError('Cannot revoke using a personal access token') elif isinstance(auth, AccessTokenAuthentication): - self._service.revoke_personal_access_tokens(auth) + with self._rejecting_unusable_token(auth): + self._service.revoke_personal_access_tokens(auth) return Response(status_code=204, body='') else: raise UnauthorizedError('Valid access token required') + + @contextmanager + def _rejecting_unusable_token(self, + auth: AccessTokenAuthentication + ) -> Iterator[None]: + """ + Translate the ways in which an access token can turn out to be unusable + into the 401 response both endpoints document for that case. An + unexpected response from the authorization server is not one of them; + it raises, and the resulting 5xx trips an alarm, as it should. + """ + try: + yield + except (InvalidAccessTokenError, + ForeignTokenException, + UnknownUserException) as e: + log.warning('Rejecting %s', auth, exc_info=e) + raise UnauthorizedError('Valid access token required') diff --git a/src/azul/terra.py b/src/azul/terra.py index 3e1800ba7c..ecea6afc95 100644 --- a/src/azul/terra.py +++ b/src/azul/terra.py @@ -106,6 +106,7 @@ from azul.oauth2 import ( CredentialedClient, CredentialsProvider, + InvalidAccessTokenError, ServiceAccountCredentials, TokenCredentials, ) @@ -677,6 +678,9 @@ def for_registered_user(cls, authentication: BearerTokenAuthentication) -> Self: self = cls(credentials_provider=UserCredentialsProvider(authentication)) try: self.validate() + except InvalidAccessTokenError as e: + log.warning('Invalid credentials', exc_info=e) + raise UnauthorizedError('Invalid credentials') except AssertionError as e: if R.caused(e): log.warning('Invalid credentials', exc_info=e) diff --git a/test/service/test_user_controller.py b/test/service/test_user_controller.py index a70215344e..3c16476966 100644 --- a/test/service/test_user_controller.py +++ b/test/service/test_user_controller.py @@ -5,6 +5,9 @@ contextmanager, ) import json +from typing import ( + ContextManager, +) from unittest.mock import ( PropertyMock, patch, @@ -32,6 +35,7 @@ HasCachedHttpClient, ) from azul.lib import ( + R, cached_property, ) from azul.logging import ( @@ -39,6 +43,7 @@ get_test_logger, ) from azul.oauth2 import ( + InvalidAccessTokenError, OAuth2Client, TokenForCodeResponse, TokenInfoResponse, @@ -125,11 +130,11 @@ def setUp(self): UserService._apat_public_key.fdel(self._service) @contextmanager - def _mock_token_info(self): + def _mock_token_info(self, aud: str = 'mock_client_id'): with patch.object(OAuth2Client, 'token_info') as mock: mock.return_value = TokenInfoResponse( - azp='mock_client_id', - aud='mock_client_id', + azp=aud, + aud=aud, sub=self._mock_sub, scope='https://www.googleapis.com/auth/userinfo.email openid', exp='9999999999', @@ -433,6 +438,47 @@ def test_revoke_unauthenticated(self): response = self._revoke() self.assertEqual(401, response.status) + def test_unexpected_token_info_response(self): + # A response from the authorization server that we aren't prepared to + # interpret is not necessarily the client's fault, and we don't know + # what to make of it. The resulting 5xx trips an alarm, which is how + # we'd learn that it needs handling. + token = self._mock_access_token + error = AssertionError(R('Unexpected response status', 503)) + with patch.object(OAuth2Client, 'token_info', side_effect=error): + for response in self._get_token(token), self._revoke(token): + self.assertEqual(500, response.status) + + def test_unusable_access_token(self): + token = self._mock_access_token + # The user of a foreign token is never loaded, and the user of the + # token in the third case was never authorized, so neither case + # reaches the point of needing a user + contexts: list[tuple[str, ContextManager]] = [ + ( + 'rejected by the authorization server', + patch.object(OAuth2Client, 'token_info', side_effect=InvalidAccessTokenError) + ), + ( + 'issued to another client', + self._mock_token_info(aud='other_client_id') + ), + ( + 'issued to an unknown user', + self._mock_token_info() + ) + ] + for description, context in contexts: + with self.subTest(description): + with context: + for response in self._get_token(token), self._revoke(token): + self.assertEqual(401, response.status) + expected_body = { + 'Code': 'UnauthorizedError', + 'Message': 'Valid access token required' + } + self.assertEqual(expected_body, json.loads(response.data)) + def test_unsupported_endpoint_rejects_pat(self): apat = self._authorize_and_mint() url = str(self.base_url.set(path='/index/summary')) From 9183fe6fca949397a759ba8f7376c1186f1bd47a Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Thu, 8 Oct 2026 15:44:28 -0700 Subject: [PATCH 09/11] Reject an unusable Authorization header with 401, without echoing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The message of the raised `UnauthorizedError` repeated the header, so the response carried the client's own credentials back to it. A bearer token of unexpected syntax yielded 400, at two sites, where RFC 6750 §3.1 classes a malformed token under `invalid_token`, to be answered with a 401. Co-Authored-By: Claude Opus 5 (1M context) --- lambdas/service/app.py | 7 +++---- src/azul/auth.py | 4 ++-- test/service/test_user_controller.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/lambdas/service/app.py b/lambdas/service/app.py index 5bd20710eb..f7271fd72d 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -1,7 +1,6 @@ import logging.config from chalice import ( - BadRequestError, UnauthorizedError, ) @@ -266,15 +265,15 @@ def _authenticate(self) -> BearerTokenAuthentication | None: try: auth_type, auth_token = header.split() except ValueError: - raise UnauthorizedError(header) + raise UnauthorizedError('Malformed Authorization header') else: if auth_type.lower() == 'bearer': if not is_redactable(auth_token): - raise BadRequestError('Unexpected token syntax') + raise UnauthorizedError('Unexpected token syntax') else: return BearerTokenAuthentication.for_token(auth_token) else: - raise UnauthorizedError(header) + raise UnauthorizedError('Unsupported authorization scheme') app = ServiceApp() diff --git a/src/azul/auth.py b/src/azul/auth.py index e82004b89a..723fccbd33 100644 --- a/src/azul/auth.py +++ b/src/azul/auth.py @@ -8,7 +8,7 @@ import attr from chalice.app import ( - BadRequestError, + UnauthorizedError, ) from azul import ( @@ -62,7 +62,7 @@ def for_token(cls, token: str) -> BearerTokenAuthentication: elif looks_like_access_token(token): return AccessTokenAuthentication(token) else: - raise BadRequestError('Unexpected token syntax') + raise UnauthorizedError('Unexpected token syntax') def identity(self) -> str: return self.token diff --git a/test/service/test_user_controller.py b/test/service/test_user_controller.py index 3c16476966..5a7dd5791b 100644 --- a/test/service/test_user_controller.py +++ b/test/service/test_user_controller.py @@ -438,6 +438,25 @@ def test_revoke_unauthenticated(self): response = self._revoke() self.assertEqual(401, response.status) + def test_unusable_authorization_header(self): + url = str(self.base_url.set(path='/user/token')) + # A refresh token is redactable, so it passes the syntax check in the + # app and is rejected by `BearerTokenAuthentication.for_token` instead + refresh_token = self._mock_refresh_token() + for description, header, message in [ + ('no scheme', self._mock_access_token, 'Malformed Authorization header'), + ('other scheme', 'Basic dXNlcjpwYXNz', 'Unsupported authorization scheme'), + ('unredactable token', 'Bearer not_a_token', 'Unexpected token syntax'), + ('token of another kind', f'Bearer {refresh_token}', 'Unexpected token syntax') + ]: + with self.subTest(description): + headers = {'Authorization': header} + response = self._http_client.request('GET', url, headers=headers) + self.assertEqual(401, response.status) + body = response.data.decode() + self.assertEqual(message, json.loads(body)['Message']) + self.assertNotIn(header, body) + def test_unexpected_token_info_response(self): # A response from the authorization server that we aren't prepared to # interpret is not necessarily the client's fault, and we don't know From bb05be9aafd1eca6eb7811154f366ab12de7d924 Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Wed, 21 Oct 2026 21:54:24 -0700 Subject: [PATCH 10/11] Bind the local Chalice server in the IT to an ephemeral port The server bound port 8000, so a second, concurrent run of the IT failed in `setUpClass` with `Address already in use`. That port is also the default of `chalice local`, which `make -C lambdas/service local` runs, so a locally served app collided with the IT just the same. The operating system now picks the port, which is read back from the bound socket, as `LocalAppTestCase` already does. The endpoint is therefore only known once the server exists, leaving `url` a declaration. The `None` that `server` and `server_thread` were initialized with was never observed, since `tearDownClass` is skipped when `setUpClass` raises, so they become declarations, too. Co-Authored-By: Claude Opus 5 (1M context) --- test/integration_test.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/integration_test.py b/test/integration_test.py index 5eb44964ca..aa5caa44f3 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -61,6 +61,9 @@ UnauthorizedError, ) import chalice.cli +from chalice.local import ( + LocalDevServer, +) import fastavro from furl import ( furl, @@ -2387,9 +2390,9 @@ def test_openapi(self): class AzulChaliceLocalIntegrationTest(IntegrationTestCase): - url = furl(scheme='http', host='127.0.0.1', port=8000) - server = None - server_thread = None + url: ClassVar[furl] + server: ClassVar[LocalDevServer] + server_thread: ClassVar[threading.Thread] @classmethod def setUpClass(cls) -> None: @@ -2400,8 +2403,10 @@ def setUpClass(cls) -> None: config = factory.create_config_obj() cls.server = factory.create_local_server(app_obj=app_module.app, config=config, - host=cls.url.host, - port=cls.url.port) + host='127.0.0.1', + port=0) + host, port = cls.server.server.server_address + cls.url = furl(scheme='http', host=host, port=port) cls.server_thread = threading.Thread(target=cls.server.server.serve_forever) cls.server_thread.start() From 0a35fc85972ce01d5f683a5acd133d3dfd5858ff Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Thu, 22 Oct 2026 08:54:47 -0700 Subject: [PATCH 11/11] Reject a personal access token with 403 instead of 400 The credential is valid and was authenticated; the operation is merely not available to it. Both endpoints document the status now, as they already did the 401. Co-Authored-By: Claude Opus 5 (1M context) --- lambdas/service/openapi.json | 6 ++++++ src/azul/service/user_controller.py | 17 +++++++++++++++-- test/service/test_user_controller.py | 4 ++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json index 335329e5e0..84020b807c 100644 --- a/lambdas/service/openapi.json +++ b/lambdas/service/openapi.json @@ -14043,6 +14043,9 @@ }, "401": { "description": "\nNo valid OAuth 2.0 access token was provided\n" + }, + "403": { + "description": "\nA personal access token was provided, which cannot\nbe exchanged for another one\n" } }, "summary": "Obtain a personal access token", @@ -14078,6 +14081,9 @@ }, "401": { "description": "\nNo valid OAuth 2.0 access token was provided\n" + }, + "403": { + "description": "\nA personal access token was provided, which cannot\nbe used to revoke\n" } }, "summary": "Revoke all personal access tokens", diff --git a/src/azul/service/user_controller.py b/src/azul/service/user_controller.py index 6a17cd0017..9d77852797 100644 --- a/src/azul/service/user_controller.py +++ b/src/azul/service/user_controller.py @@ -16,6 +16,7 @@ from chalice.app import ( BadRequestError, + ForbiddenError, Response, UnauthorizedError, ) @@ -195,6 +196,12 @@ def authorize(): 'description': fd(''' No valid OAuth 2.0 access token was provided ''') + }, + '403': { + 'description': fd(''' + A personal access token was provided, which cannot + be exchanged for another one + ''') } } } @@ -226,6 +233,12 @@ def token(): 'description': fd(''' No valid OAuth 2.0 access token was provided ''') + }, + '403': { + 'description': fd(''' + A personal access token was provided, which cannot + be used to revoke + ''') } } } @@ -327,7 +340,7 @@ def _authorize(self) -> JSON: def _token(self) -> JSON: auth = self._authentication(self.current_request) if isinstance(auth, PersonalAccessTokenAuthentication): - raise BadRequestError('Cannot exchange a personal access token for another') + raise ForbiddenError('Cannot exchange a personal access token for another') elif isinstance(auth, AccessTokenAuthentication): with self._rejecting_unusable_token(auth): apat_auth = self._service.mint_personal_access_token(auth) @@ -338,7 +351,7 @@ def _token(self) -> JSON: def _revoke(self) -> Response: auth = self._authentication(self.current_request) if isinstance(auth, PersonalAccessTokenAuthentication): - raise BadRequestError('Cannot revoke using a personal access token') + raise ForbiddenError('Cannot revoke using a personal access token') elif isinstance(auth, AccessTokenAuthentication): with self._rejecting_unusable_token(auth): self._service.revoke_personal_access_tokens(auth) diff --git a/test/service/test_user_controller.py b/test/service/test_user_controller.py index 5a7dd5791b..53e8f2e132 100644 --- a/test/service/test_user_controller.py +++ b/test/service/test_user_controller.py @@ -408,7 +408,7 @@ def test_token_rejects_pat(self): auth = AccessTokenAuthentication(self._mock_access_token) apat = self._service.mint_personal_access_token(auth) response = self._get_token(apat.token) - self.assertEqual(400, response.status) + self.assertEqual(403, response.status) def test_token_unauthenticated(self): response = self._get_token() @@ -432,7 +432,7 @@ def test_revoke_invalidates_apat(self): def test_revoke_rejects_pat(self): apat = self._authorize_and_mint() response = self._revoke(apat.token) - self.assertEqual(400, response.status) + self.assertEqual(403, response.status) def test_revoke_unauthenticated(self): response = self._revoke()