From 71bb6570150aa4df6f7075469cf68b8d7f9f652e Mon Sep 17 00:00:00 2001 From: Qiheng He Date: Fri, 23 Oct 2026 14:29:28 +0800 Subject: [PATCH 1/2] Comment the memory of every implementation on pull requests The workflow of each implementation uploads the numbers of its measure scenario as the artifact memory-. shared/tools/measure.sh writes them to the file in VFOX_PLUS_MEASURE_OUTPUT as well as to the log; Flutter's tool/measure.py already wrote memory.txt. memory.yml starts after each of those workflows ends, in the context of main, so that it also works for pull requests of forks, and .github/scripts/memory_report.py stops while another one is still running for the commit. The run after the last one writes one comment on the pull request: the table of docs/memory.md, sorted by the PSS of the Home page, with the difference from the latest run of the same workflow on main. An implementation that the pull request did not rebuild shows the numbers of main. Later commits replace the comment instead of adding another one. The same code does not use the same memory in every CI run. In the 107 runs of 2026-10-18 and 19 that shared their source with another one (30 groups of 2 to 8 runs), the PSS differed by up to 4.3 % for mygo, Uno, Capacitor, Tauri, Dioxus and Wails, by up to 7.5 % for Compose and Flet, and by 11 to 13 % for Gio and Flutter. A difference is a change, in bold, only when it is more than 5 MB and 5 % of the number on main (8 % for Compose and Flet, 12 % for Gio, 15 % for Flutter); none of the 940 comparisons of identical code was one by these limits. A different number of processes is a change too: it never varied. The script reads nothing but two lines of numbers from the artifacts of the pull request, which are not trusted. Its tests (83, run by the Repository workflow) use a fake of the API, and were also checked against the real one with read-only requests. --- .github/scripts/memory_report.py | 686 ++++++++++++++++++++ .github/scripts/test_memory_report.py | 878 ++++++++++++++++++++++++++ .github/workflows/capacitor.yml | 9 + .github/workflows/compose.yml | 9 + .github/workflows/dioxus.yml | 9 + .github/workflows/flet.yml | 9 + .github/workflows/flutter.yml | 7 + .github/workflows/gio.yml | 9 + .github/workflows/memory.yml | 86 +++ .github/workflows/mygo.yml | 9 + .github/workflows/repository.yml | 3 + .github/workflows/tauri.yml | 9 + .github/workflows/uno.yml | 9 + .github/workflows/wails.yml | 9 + .gitignore | 5 + CONTRIBUTING.md | 9 +- docs/memory.md | 72 +++ docs/new-implementation.md | 7 + shared/tools/measure.sh | 16 +- 19 files changed, 1847 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/memory_report.py create mode 100644 .github/scripts/test_memory_report.py create mode 100644 .github/workflows/memory.yml diff --git a/.github/scripts/memory_report.py b/.github/scripts/memory_report.py new file mode 100644 index 0000000..cbb48ef --- /dev/null +++ b/.github/scripts/memory_report.py @@ -0,0 +1,686 @@ +#!/usr/bin/env python3 +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Comments on a pull request with the memory that every implementation uses. + +The workflow of each implementation (.github/workflows/.yml) +measures its application with shared/tools/measure.sh (Flutter: +flutter/tool/measure.py) and uploads the result as the artifact +memory-, which holds one file, memory.txt, with the lines +"memory (): RSS MB, PSS MB in processes" of the points home +and nodejs. .github/workflows/memory.yml runs this script when one of those +workflows ends. Once all of them have ended for the latest commit of the pull +request, the script writes one comment: a table like the one of docs/memory.md, +sorted by the PSS of the Home page, with the difference from the latest +measurement on the default branch next to the numbers. A later run replaces +that comment instead of adding another one. + +An implementation that the pull request did not rebuild, because its workflow +only runs for changes in its own paths, shows the numbers of the default branch. + +Usage: + + memory_report.py --run RUN_ID [--dry-run] + Reports on the pull request of the workflow run RUN_ID, if that was the + last of the implementation workflows to end for the commit. + + memory_report.py --pull-request NUMBER [--dry-run] + Reports on that pull request now, with the implementations that are still + running marked as such. + +--dry-run prints the comment instead of writing it. The implementations come +from --implementations or MEMORY_IMPLEMENTATIONS (names separated by spaces). +Needs the gh CLI, which GH_TOKEN authenticates. The artifacts come from the +pull request, so they are not trusted: only the numbers in the lines above +are read from them. +""" + +import argparse +import io +import json +import os +import re +import subprocess +import sys +import time +import urllib.parse +import zipfile +from collections import defaultdict +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +# The first line of the comment, which finds it again. +MARKER = "" +# The author of the comments that GITHUB_TOKEN writes. +BOT = "github-actions[bot]" +WORKFLOWS = ".github/workflows" +MEMORY_FILE = "memory.txt" +# memory.txt has two short lines; an artifact much larger than that is not one. +MAX_ARTIFACT_BYTES = 64 * 1024 +MAX_MEMORY_FILE_BYTES = 4 * 1024 + +# The same code does not use the same memory in every CI run. Comparing every +# two of the 107 runs on 2026-10-18 and 19 that shared their source with another +# one (30 groups of 2 to 8 runs of the ten implementations; see docs/memory.md), +# the Home and the Versions PSS never differed by more than 5 MB and 5 % of the +# first number for most of the implementations, and never by more than the +# percentages below for the four others. Only a larger difference counts as a +# change. +MIN_CHANGE_MB = 5 +DEFAULT_TOLERANCE = 0.05 +TOLERANCES = {"compose": 0.08, "flet": 0.08, "gio": 0.12, "flutter": 0.15} + +LINE = re.compile(r"memory \((home|nodejs)\): RSS ([0-9]{1,6}) MB, PSS ([0-9]{1,6}) MB in ([0-9]{1,4}) processes") +IMPLEMENTATION = re.compile(r"[a-z0-9][a-z0-9-]*") +REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") + + +@dataclass(frozen=True) +class Point: + """The memory at one measuring point, in MB.""" + + rss: int + pss: int + processes: int + + +@dataclass(frozen=True) +class Memory: + """What an implementation uses on the Home page and with the versions of Node.js listed.""" + + home: Point + versions: Point + + +@dataclass +class Row: + """One implementation of the report. + + The state is "measured" (the pull request rebuilt it), "inherited" (it did + not, so the numbers are those of the default branch), "running" or "failed" + (the workflow of the pull request has not ended, or ended without numbers) + or "missing" (no workflow ran and the default branch has no numbers). + """ + + implementation: str + name: str + technology: str = "" + state: str = "missing" + memory: Memory | None = None + baseline: Memory | None = None + run_url: str = "" + baseline_url: str = "" + baseline_sha: str = "" + + @property + def shown(self) -> Memory | None: + """The numbers of the table.""" + if self.state == "measured": + return self.memory + if self.state == "inherited": + return self.baseline + return None + + +class ApiError(RuntimeError): + """A request to the API of GitHub failed.""" + + +def parse_memory(text: str) -> Memory | None: + """Parses memory.txt: one line for the point home and one for nodejs, nothing else.""" + points: dict[str, Point] = {} + for line in text.splitlines(): + line = line.strip() + if not line: + continue + match = LINE.fullmatch(line) + if match is None or match[1] in points: + return None + points[match[1]] = Point(rss=int(match[2]), pss=int(match[3]), processes=int(match[4])) + if points.keys() != {"home", "nodejs"}: + return None + return Memory(home=points["home"], versions=points["nodejs"]) + + +def tolerance(implementation: str, baseline: int) -> float: + """The difference in MB from the baseline that is still the noise of CI.""" + return max(MIN_CHANGE_MB, TOLERANCES.get(implementation, DEFAULT_TOLERANCE) * baseline) + + +def exceeds_noise(implementation: str, baseline: int, current: int) -> bool: + """Whether the difference is a change and not the variation between CI runs.""" + return abs(current - baseline) > tolerance(implementation, baseline) + + +def signed(number: int) -> str: + return f"+{number}" if number > 0 else str(number) + + +def cell(text: str) -> str: + """Text that cannot break out of a table cell.""" + return re.sub(r"\s+", " ", text).replace("|", "\\|").strip() + + +def read_technologies(text: str) -> dict[str, str]: + """The Technology column of the first table of docs/memory.md, by implementation directory.""" + technologies: dict[str, str] = {} + columns: tuple[int, int] | None = None + for line in text.splitlines(): + if not line.startswith("|"): + if columns is not None: + break + continue + cells = [part.strip() for part in line.strip().strip("|").split("|")] + if columns is None: + if "Implementation" in cells and "Technology" in cells: + columns = (cells.index("Implementation"), cells.index("Technology")) + continue + if len(cells) <= max(columns): + continue + link = re.search(r"\]\(\.\./([A-Za-z0-9_-]+)/README\.md", cells[columns[0]]) + if link: + technologies[link[1]] = cells[columns[1]] + return technologies + + +def pss_cell(row: Row, point: str) -> str: + """The PSS of a point, with the difference from the default branch if it was measured.""" + shown = getattr(row.shown, point).pss + if row.state != "measured" or row.baseline is None: + return str(shown) + before = getattr(row.baseline, point).pss + difference = signed(shown - before) + if exceeds_noise(row.implementation, before, shown): + difference = f"**{difference}**" + return f"{shown} ({difference})" + + +def table_cells(row: Row) -> list[str]: + name = row.name + if row.state == "inherited": + name += " (main)" + elif row.state == "measured" and row.baseline is None: + name += " (no baseline)" + shown = row.shown + if shown is None: + label = {"running": "running", "failed": "no result", "missing": "no data"}[row.state] + if row.run_url: + label = f"[{label}]({row.run_url})" + return [cell(name), cell(row.technology), "-", label, "-", "-", "-", "-"] + processes = str(shown.home.processes) + if row.state == "measured" and row.baseline is not None: + # The number of processes of an implementation does not vary between CI runs. + difference = shown.home.processes - row.baseline.home.processes + if difference: + processes += f" (**{signed(difference)}**)" + return [ + cell(name), + cell(row.technology), + processes, + pss_cell(row, "home"), + str(shown.home.rss), + pss_cell(row, "versions"), + str(shown.versions.rss), + signed(shown.versions.pss - shown.home.pss), + ] + + +def sort_key(row: Row) -> tuple[int, int, str]: + group = {"measured": 0, "inherited": 0, "running": 1, "failed": 2, "missing": 3}[row.state] + shown = row.shown + return (group, shown.home.pss if shown else 0, row.name.lower()) + + +def changes_of(row: Row) -> list[str]: + """The differences from the default branch that are more than the noise.""" + if row.state != "measured" or row.memory is None or row.baseline is None: + return [] + changes = [] + for label, point in (("Home PSS", "home"), ("Versions PSS", "versions")): + before = getattr(row.baseline, point).pss + after = getattr(row.memory, point).pss + if exceeds_noise(row.implementation, before, after): + percent = round(100 * (after - before) / before) if before else 0 + changes.append(f"{label} {signed(after - before)} MB ({signed(percent)}%)") + if row.memory.home.processes != row.baseline.home.processes: + changes.append(f"processes {row.baseline.home.processes} -> {row.memory.home.processes}") + return changes + + +def summary(rows: list[Row]) -> list[str]: + """The lines above the table: what changed, and what has no numbers.""" + lines = [] + compared = [row for row in rows if row.state == "measured" and row.baseline is not None] + changed = {row.name: changes_of(row) for row in compared} + changed = {name: parts for name, parts in changed.items() if parts} + if changed: + details = "; ".join(f"{name}: {', '.join(parts)}" for name, parts in changed.items()) + lines.append(f"Changed from main beyond the noise of CI: {details}.") + same = ", ".join(row.name for row in compared if row.name not in changed) + if same and changed: + lines.append(f"Within the noise of CI: {same}.") + elif same: + lines.append(f"No change from main beyond the noise of CI in {same}.") + without_baseline = [row.name for row in rows if row.state == "measured" and row.baseline is None] + for text, names in ( + ("No numbers on main to compare with yet", without_baseline), + ("Still running", [row.name for row in rows if row.state == "running"]), + ("No result, the run failed or was cancelled", [row.name for row in rows if row.state == "failed"]), + ("No numbers anywhere", [row.name for row in rows if row.state == "missing"]), + ): + if names: + lines.append(f"{text}: {', '.join(names)}.") + if all(row.state in ("inherited", "missing") for row in rows): + lines.append("This pull request did not rebuild any implementation, so the numbers are those of main.") + return lines + + +def explanation(rows: list[Row]) -> str: + wider = defaultdict(list) + for row in rows: + if row.implementation in TOLERANCES: + wider[TOLERANCES[row.implementation]].append(row.name) + exceptions = ", ".join( + f"{percent:.0%} for {' and '.join(names)}" for percent, names in sorted(wider.items()) + ) + return ( + "The same code does not use the same memory in every CI run, so a difference counts as a change only " + f"when it is more than {MIN_CHANGE_MB} MB and more than {DEFAULT_TOLERANCE:.0%} of the number on main" + + (f" ({exceptions}, which vary more)" if exceptions else "") + + "." + ) + + +def render(rows: list[Row], *, head_sha: str) -> str: + """The comment.""" + rows = sorted(rows, key=sort_key) + header = [ + "Implementation", + "Technology", + "Processes", + "Home PSS", + "Home RSS", + "Versions PSS", + "Versions RSS", + "The list adds (PSS)", + ] + alignment = ["---", "---", *["---:"] * 6] + body = [table_cells(row) for row in rows] + if not any(row.technology for row in rows): + # Without the table of docs/memory.md there is nothing to put there. + for line in (header, alignment, *body): + del line[1] + table = ["| " + " | ".join(line) + " |" for line in (header, alignment, *body)] + + runs = [f"[{row.name}]({row.run_url})" for row in rows if row.run_url] + baselines = [ + f"[{row.name}]({row.baseline_url}) (`{row.baseline_sha[:7]}`)" for row in rows if row.baseline_url + ] + details = [ + "- Measured by `shared/tools/measure.sh` (Flutter: `flutter/tool/measure.py`) in the workflow of each " + "implementation, on a GitHub-hosted runner with Xvfb and no GPU. These are not the numbers of " + "`docs/memory.md`, which come from another machine: compare the rows of one comment.", + "- " + explanation(rows), + ] + if runs: + details.append("- Runs of this pull request: " + ", ".join(runs) + ".") + if baselines: + details.append("- Latest runs on main: " + ", ".join(baselines) + ".") + + lines = [ + MARKER, + f"### Memory use at `{head_sha[:7]}`", + "", + *[f"- {line}" for line in summary(rows)], + "", + *table, + "", + "In MB, for the application and all of its child processes, sorted by the PSS of the Home page; " + '"Versions" is the SDKs page with the available versions of Node.js listed. A number in parentheses is ' + "the difference from main, in bold when it is more than the noise of CI. " + '"(main)" marks an implementation that this pull request did not rebuild: its numbers are those of main.', + "", + "
", + "Details", + "", + *details, + "
", + "", + ] + return "\n".join(lines) + + +class GitHub: + """The REST API of one repository, through the gh CLI (GH_TOKEN authenticates it).""" + + def __init__(self, repository: str) -> None: + self.repository = repository + + def _request(self, endpoint: str, *, method: str = "GET", payload: Any = None, attempts: int = 3) -> bytes: + command = ["gh", "api", "--method", method, endpoint] + data = None + if payload is not None: + command += ["--input", "-"] + data = json.dumps(payload).encode() + for attempt in range(1, attempts + 1): + result = subprocess.run(command, input=data, capture_output=True, check=False) + if result.returncode == 0: + return result.stdout + error = result.stderr.decode(errors="replace").strip() + transient = re.search(r"HTTP 5[0-9][0-9]|timeout|timed out|connection reset|EOF", error, re.IGNORECASE) + if attempt == attempts or not transient: + raise ApiError(f"{method} {endpoint}: {error}") + time.sleep(2**attempt) + raise AssertionError("unreachable") + + def _endpoint(self, path: str, params: dict[str, Any]) -> str: + endpoint = f"repos/{self.repository}" + (f"/{path}" if path else "") + return endpoint + ("?" + urllib.parse.urlencode(params) if params else "") + + def get(self, path: str, **params: Any) -> Any: + return json.loads(self._request(self._endpoint(path, params))) + + def raw(self, path: str) -> bytes: + """The body of a response that is not JSON, such as the zip of an artifact.""" + return self._request(self._endpoint(path, {})) + + def pages(self, path: str, key: str | None = None, **params: Any) -> Iterator[Any]: + """Every item of a list, 100 at a time.""" + for page in range(1, 11): + data = self.get(path, per_page=100, page=page, **params) + items = data[key] if key else data + yield from items + if len(items) < 100: + return + + def send(self, method: str, path: str, payload: Any = None) -> Any: + # Creating a comment again after a lost response would write it twice. + attempts = 1 if method == "POST" else 3 + body = self._request(self._endpoint(path, {}), method=method, payload=payload, attempts=attempts) + return json.loads(body) if body.strip() else None + + +def log(message: str) -> None: + print(message, file=sys.stderr, flush=True) + + +def read_artifact(gh: GitHub, artifact: dict[str, Any]) -> Memory | None: + """The numbers in an artifact, or None if it is not a valid memory.txt.""" + if artifact.get("expired") or artifact.get("size_in_bytes", 0) > MAX_ARTIFACT_BYTES: + return None + try: + data = gh.raw(f"actions/artifacts/{artifact['id']}/zip") + with zipfile.ZipFile(io.BytesIO(data)) as archive: + files = archive.infolist() + if len(files) != 1 or files[0].filename != MEMORY_FILE: + return None + with archive.open(files[0]) as file: + content = file.read(MAX_MEMORY_FILE_BYTES + 1) + except (zipfile.BadZipFile, ApiError) as error: + log(f"Cannot read the artifact {artifact.get('name')} ({artifact.get('id')}): {error}") + return None + if len(content) > MAX_MEMORY_FILE_BYTES: + return None + return parse_memory(content.decode("utf-8", errors="replace")) + + +def newest(artifacts: list[dict[str, Any]]) -> dict[str, Any] | None: + live = [artifact for artifact in artifacts if not artifact.get("expired")] + return max(live, key=lambda artifact: artifact["created_at"], default=None) + + +def baseline_of(gh: GitHub, implementation: str, default_branch: str) -> tuple[Memory, dict[str, Any]] | None: + """The latest numbers of an implementation that were measured on the default branch.""" + artifacts = gh.get("actions/artifacts", name=f"memory-{implementation}", per_page=100)["artifacts"] + # A pull request from a fork can have a branch called like the default branch. + trusted = [ + artifact + for artifact in artifacts + if (artifact.get("workflow_run") or {}).get("head_branch") == default_branch + and artifact["workflow_run"].get("head_repository_id") == artifact["workflow_run"].get("repository_id") + ] + for artifact in sorted(trusted, key=lambda candidate: candidate["created_at"], reverse=True): + memory = read_artifact(gh, artifact) + if memory is not None: + return memory, artifact + return None + + +def find_pull_requests(gh: GitHub, owner: str | None, branch: str, sha: str) -> list[dict[str, Any]]: + """The open pull requests whose latest commit is sha.""" + pulls: list[dict[str, Any]] = [] + if owner: + pulls = gh.get("pulls", state="open", head=f"{owner}:{branch}", per_page=100) + if not pulls: + pulls = [pull for pull in gh.get(f"commits/{sha}/pulls", per_page=100) if pull["state"] == "open"] + return [pull for pull in pulls if pull["head"]["sha"] == sha] + + +def workflow_path(implementation: str) -> str: + return f"{WORKFLOWS}/{implementation}.yml" + + +def active_workflows(gh: GitHub, implementations: list[str]) -> dict[str, dict[str, Any]]: + """The workflows of the implementations, by implementation; one that is gone or disabled is left out.""" + workflows = {workflow["path"]: workflow for workflow in gh.get("actions/workflows", per_page=100)["workflows"]} + return { + implementation: workflows[workflow_path(implementation)] + for implementation in implementations + if workflows.get(workflow_path(implementation), {}).get("state") == "active" + } + + +def latest_runs(gh: GitHub, head_sha: str) -> dict[str, dict[str, Any]]: + """The latest run of every workflow of the pull request for a commit, by the path of the workflow.""" + runs: dict[str, dict[str, Any]] = {} + for run in gh.get("actions/runs", head_sha=head_sha, event="pull_request", per_page=100)["workflow_runs"]: + if run["path"] not in runs or run["id"] > runs[run["path"]]["id"]: + runs[run["path"]] = run + return runs + + +def unfinished( + workflows: dict[str, dict[str, Any]], runs: dict[str, dict[str, Any]], finished_run: int | None +) -> list[str]: + """The names of the workflows with a run that has not ended, other than the run that has just ended.""" + return [ + workflow["name"] + for implementation, workflow in workflows.items() + if (run := runs.get(workflow_path(implementation))) + and run["status"] != "completed" + and run["id"] != finished_run + ] + + +def collect( + gh: GitHub, + *, + workflows: dict[str, dict[str, Any]], + runs: dict[str, dict[str, Any]], + technologies: dict[str, str], + default_branch: str, + finished_run: int | None, +) -> list[Row]: + """The rows of the report: the numbers of the pull request and of the default branch.""" + rows = [] + for implementation, workflow in workflows.items(): + row = Row(implementation, workflow["name"], technologies.get(implementation, "")) + run = runs.get(workflow_path(implementation)) + if run is None: + row.state = "inherited" + else: + row.run_url = run["html_url"] + if run["status"] != "completed" and run["id"] != finished_run: + row.state = "running" + else: + artifacts = gh.get( + f"actions/runs/{run['id']}/artifacts", name=f"memory-{implementation}", per_page=100 + )["artifacts"] + artifact = newest(artifacts) + row.memory = read_artifact(gh, artifact) if artifact else None + row.state = "measured" if row.memory else "failed" + if row.state in ("measured", "inherited"): + baseline = baseline_of(gh, implementation, default_branch) + if baseline is not None: + memory, artifact = baseline + row.baseline = memory + row.baseline_sha = artifact["workflow_run"]["head_sha"] + row.baseline_url = f"https://github.com/{gh.repository}/actions/runs/{artifact['workflow_run']['id']}" + elif row.state == "inherited": + row.state = "missing" + rows.append(row) + return rows + + +def same_text(left: str, right: str) -> bool: + return left.replace("\r\n", "\n").strip() == right.replace("\r\n", "\n").strip() + + +def upsert_comment(gh: GitHub, number: int, body: str) -> str: + """Writes the comment of the pull request, or replaces the one that an earlier run wrote.""" + mine = sorted( + ( + comment + for comment in gh.pages(f"issues/{number}/comments") + if (comment.get("user") or {}).get("login") == BOT and comment["body"].startswith(MARKER) + ), + key=lambda comment: comment["id"], + ) + if not mine: + gh.send("POST", f"issues/{number}/comments", {"body": body}) + return "created" + result = "unchanged" + if not same_text(mine[0]["body"], body): + gh.send("PATCH", f"issues/comments/{mine[0]['id']}", {"body": body}) + result = "updated" + # Two runs at the same time can both have created one. + for extra in mine[1:]: + gh.send("DELETE", f"issues/comments/{extra['id']}") + return result + + +def report( + gh: GitHub, + *, + implementations: list[str], + technologies: dict[str, str], + run_id: int | None, + pull_request: int | None, + dry_run: bool, +) -> int: + # Most runs end here, on the way to the one that finds that all of them have: keep it to a few requests. + if run_id is not None: + run = gh.get(f"actions/runs/{run_id}") + if run["event"] != "pull_request": + log(f"The run {run_id} is a {run['event']} run, not one of a pull request.") + return 0 + head_sha = run["head_sha"] + owner = ((run.get("head_repository") or {}).get("owner") or {}).get("login") + pulls = find_pull_requests(gh, owner, run["head_branch"], head_sha) + else: + pull = gh.get(f"pulls/{pull_request}") + if pull["state"] != "open": + log(f"The pull request {pull_request} is not open.") + return 0 + head_sha = pull["head"]["sha"] + pulls = [pull] + if not pulls: + log(f"No open pull request has {head_sha[:7]} as its latest commit; there is nothing to report on.") + return 0 + workflows = active_workflows(gh, implementations) + if not workflows: + log("None of the implementations has a workflow.") + return 0 + runs = latest_runs(gh, head_sha) + waiting = unfinished(workflows, runs, run_id) if run_id is not None else [] + if waiting: + log(f"Waiting for {', '.join(waiting)}; the report is written when they have ended.") + return 0 + + rows = collect( + gh, + workflows=workflows, + runs=runs, + technologies=technologies, + default_branch=gh.get("")["default_branch"], + finished_run=run_id, + ) + body = render(rows, head_sha=head_sha) + summary_file = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_file: + with open(summary_file, "a", encoding="utf-8") as file: + file.write(body.removeprefix(MARKER + "\n")) + if dry_run: + print(body) + return 0 + for pull in pulls: + # A commit that came while the numbers were collected has its own report. + current = gh.get(f"pulls/{pull['number']}") + if current["state"] != "open" or current["head"]["sha"] != head_sha: + log(f"The pull request {pull['number']} has moved on from {head_sha[:7]}.") + continue + log(f"Pull request {pull['number']}: comment {upsert_comment(gh, pull['number'], body)}.") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", ""), help="OWNER/NAME") + parser.add_argument( + "--implementations", + default=os.environ.get("MEMORY_IMPLEMENTATIONS", ""), + help="the implementations, such as 'mygo tauri', separated by spaces", + ) + parser.add_argument("--docs", type=Path, default=Path("docs/memory.md"), help="the page with the Technology column") + parser.add_argument("--dry-run", action="store_true", help="print the comment instead of writing it") + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument("--run", type=int, help="the ID of the workflow run that has just ended") + target.add_argument("--pull-request", type=int, help="the number of the pull request") + arguments = parser.parse_args(argv) + + implementations = arguments.implementations.split() + if not REPOSITORY.fullmatch(arguments.repository): + parser.error("--repository (or GITHUB_REPOSITORY) must be OWNER/NAME") + if not implementations or not all(IMPLEMENTATION.fullmatch(name) for name in implementations): + parser.error("--implementations (or MEMORY_IMPLEMENTATIONS) must list names such as 'mygo tauri'") + technologies = {} + if arguments.docs.is_file(): + technologies = read_technologies(arguments.docs.read_text(encoding="utf-8")) + try: + return report( + GitHub(arguments.repository), + implementations=implementations, + technologies=technologies, + run_id=arguments.run, + pull_request=arguments.pull_request, + dry_run=arguments.dry_run, + ) + except ApiError as error: + if "Resource not accessible by integration" in str(error): + # A token that cannot write, such as that of a run for a bot's pull request: the report is + # already in the summary of the run, which is as far as it can go. + message = f"The token of this run cannot write the comment; the report is in the summary: {error}" + # The encoding of the data of a workflow command. + message = message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + print(f"::warning::{message}") + return 0 + log(f"memory_report.py: {error}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test_memory_report.py b/.github/scripts/test_memory_report.py new file mode 100644 index 0000000..51b8667 --- /dev/null +++ b/.github/scripts/test_memory_report.py @@ -0,0 +1,878 @@ +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests of memory_report.py: python3 -m unittest discover -s .github/scripts""" + +import contextlib +import io +import os +import re +import subprocess +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + +import memory_report as report +from memory_report import Memory, Point, Row + +REPOSITORY = "version-fox/vfox-plus" +HEAD = "a" * 40 +MAIN = "b" * 40 + + +def setUpModule(): + # CI sets these. The tests must not write to the summary of the job that runs them, or + # take the repository and the implementations from it. + patcher = mock.patch.dict(os.environ) + patcher.start() + unittest.addModuleCleanup(patcher.stop) + for name in ("GITHUB_STEP_SUMMARY", "GITHUB_REPOSITORY", "MEMORY_IMPLEMENTATIONS", "GH_TOKEN"): + os.environ.pop(name, None) + + +def memory_text(home=(55, 42, 1), versions=(64, 49, 1)) -> str: + return ( + f"memory (home): RSS {home[0]} MB, PSS {home[1]} MB in {home[2]} processes\n" + f"memory (nodejs): RSS {versions[0]} MB, PSS {versions[1]} MB in {versions[2]} processes\n" + ) + + +def make_zip(files: dict[str, str | bytes]) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in files.items(): + archive.writestr(name, content) + return buffer.getvalue() + + +def memory(home_pss=100, versions_pss=120, processes=1) -> Memory: + return Memory(Point(home_pss + 10, home_pss, processes), Point(versions_pss + 10, versions_pss, processes)) + + +class FakeGitHub: + """The parts of the API of GitHub that the report reads and writes.""" + + repository = REPOSITORY + + def __init__(self) -> None: + self.default_branch = "main" + self.workflows: list[dict] = [] + self.runs: list[dict] = [] + self.artifacts: list[dict] = [] + self.zips: dict[int, bytes] = {} + self.pulls: dict[int, dict] = {} + self.comments: dict[int, list[dict]] = {} + self.writes: list[tuple[str, str]] = [] + self._ids = 1000 + + def next_id(self) -> int: + self._ids += 1 + return self._ids + + # What the tests set up. + def add_workflow(self, implementation: str, name: str | None = None, state: str = "active") -> None: + self.workflows.append( + { + "path": f".github/workflows/{implementation}.yml", + "name": name or implementation.capitalize(), + "state": state, + } + ) + + def add_pull_request(self, number: int = 7, sha: str = HEAD, state: str = "open") -> None: + head = {"sha": sha, "ref": "feature", "owner": "contributor"} + self.pulls[number] = {"number": number, "state": state, "head": head} + + def add_run( + self, + implementation: str, + text: str | bytes | None = None, + *, + status: str = "completed", + conclusion: str | None = "success", + sha: str = HEAD, + event: str = "pull_request", + run_id: int | None = None, + artifact_files: dict | None = None, + ) -> int: + run_id = run_id or self.next_id() + self.runs.append( + { + "id": run_id, + "path": f".github/workflows/{implementation}.yml", + "name": implementation.capitalize(), + "event": event, + "status": status, + "conclusion": conclusion, + "head_sha": sha, + "head_branch": "feature", + "html_url": f"https://github.com/{REPOSITORY}/actions/runs/{run_id}", + "head_repository": {"owner": {"login": "contributor"}}, + } + ) + if text is not None or artifact_files is not None: + files = artifact_files if artifact_files is not None else {"memory.txt": text} + self.add_artifact(implementation, files, run_id=run_id, branch="feature", sha=sha) + return run_id + + def add_artifact( + self, + implementation: str, + files: dict | str, + *, + run_id: int | None = None, + branch: str = "main", + sha: str = MAIN, + created: str = "2026-10-20T10:00:00Z", + expired: bool = False, + head_repository_id: int = 1, + size: int | None = None, + ) -> int: + files = {"memory.txt": files} if isinstance(files, str) else files + archive = make_zip(files) + artifact_id = self.next_id() + self.zips[artifact_id] = archive + self.artifacts.append( + { + "id": artifact_id, + "name": f"memory-{implementation}", + "size_in_bytes": len(archive) if size is None else size, + "expired": expired, + "created_at": created, + "workflow_run": { + "id": run_id or self.next_id(), + "head_branch": branch, + "head_sha": sha, + "head_repository_id": head_repository_id, + "repository_id": 1, + }, + } + ) + return artifact_id + + # The API. + def get(self, path: str, **params): + if path == "": + return {"default_branch": self.default_branch} + if path == "actions/workflows": + return {"workflows": self.workflows} + if path == "actions/runs": + return { + "workflow_runs": [ + run + for run in self.runs + if run["head_sha"] == params["head_sha"] and run["event"] == params["event"] + ] + } + if match := re.fullmatch(r"actions/runs/(\d+)", path): + return next(run for run in self.runs if run["id"] == int(match[1])) + if match := re.fullmatch(r"actions/runs/(\d+)/artifacts", path): + return { + "artifacts": [ + artifact + for artifact in self.artifacts + if artifact["workflow_run"]["id"] == int(match[1]) and artifact["name"] == params["name"] + ] + } + if path == "actions/artifacts": + return {"artifacts": [a for a in self.artifacts if a["name"] == params["name"]]} + if path == "pulls": + owner, _, branch = params["head"].partition(":") + return [ + pull + for pull in self.pulls.values() + if pull["state"] == params["state"] and pull["head"]["owner"] == owner and pull["head"]["ref"] == branch + ] + if match := re.fullmatch(r"pulls/(\d+)", path): + return self.pulls[int(match[1])] + if match := re.fullmatch(r"commits/(\w+)/pulls", path): + return [pull for pull in self.pulls.values() if pull["head"]["sha"] == match[1]] + raise AssertionError(f"Unexpected GET {path} {params}") + + def pages(self, path: str, key: str | None = None, **params): + match = re.fullmatch(r"issues/(\d+)/comments", path) + assert match, path + yield from list(self.comments.get(int(match[1]), [])) + + def raw(self, path: str) -> bytes: + match = re.fullmatch(r"actions/artifacts/(\d+)/zip", path) + assert match, path + return self.zips[int(match[1])] + + def send(self, method: str, path: str, payload=None): + self.writes.append((method, path)) + if match := re.fullmatch(r"issues/(\d+)/comments", path): + comment = {"id": self.next_id(), "body": payload["body"], "user": {"login": report.BOT}} + self.comments.setdefault(int(match[1]), []).append(comment) + return comment + if match := re.fullmatch(r"issues/comments/(\d+)", path): + for comments in self.comments.values(): + for comment in list(comments): + if comment["id"] == int(match[1]): + if method == "PATCH": + comment["body"] = payload["body"] + else: + comments.remove(comment) + return None + raise AssertionError(f"Unexpected {method} {path}") + + +class ParseMemoryTest(unittest.TestCase): + def test_reads_both_points(self): + self.assertEqual( + report.parse_memory(memory_text((55, 42, 1), (64, 49, 3))), + Memory(Point(55, 42, 1), Point(64, 49, 3)), + ) + + def test_the_order_and_blank_lines_do_not_matter(self): + text = "\n" + memory_text().splitlines()[1] + "\n\n" + memory_text().splitlines()[0] + "\n" + self.assertIsNotNone(report.parse_memory(text)) + + def test_needs_both_points(self): + self.assertIsNone(report.parse_memory(memory_text().splitlines()[0])) + self.assertIsNone(report.parse_memory("")) + + def test_rejects_a_point_twice(self): + line = memory_text().splitlines()[0] + self.assertIsNone(report.parse_memory(f"{line}\n{line}\n{memory_text().splitlines()[1]}")) + + def test_rejects_anything_else(self): + self.assertIsNone(report.parse_memory(memory_text() + "e2e: MEASURE home\n")) + self.assertIsNone(report.parse_memory(memory_text().replace("MB, PSS", "MB, PSS -"))) + self.assertIsNone(report.parse_memory(memory_text().replace("home", "other"))) + self.assertIsNone(report.parse_memory(memory_text((10**6, 1, 1), (1, 1, 1)))) + + def test_rejects_digits_that_are_not_ascii(self): + self.assertIsNone(report.parse_memory(memory_text().replace("55", "٥٥"))) + + +class ToleranceTest(unittest.TestCase): + def test_a_small_implementation_is_judged_by_the_absolute_floor(self): + self.assertFalse(report.exceeds_noise("mygo", 42, 47)) + self.assertTrue(report.exceeds_noise("mygo", 42, 48)) + self.assertTrue(report.exceeds_noise("mygo", 42, 36)) + self.assertFalse(report.exceeds_noise("mygo", 42, 37)) + + def test_a_large_implementation_is_judged_by_the_percentage(self): + self.assertFalse(report.exceeds_noise("tauri", 300, 315)) + self.assertTrue(report.exceeds_noise("tauri", 300, 316)) + + def test_the_noisy_implementations_get_more_room(self): + self.assertFalse(report.exceeds_noise("flutter", 350, 350 + 52)) + self.assertTrue(report.exceeds_noise("flutter", 350, 350 + 53)) + self.assertFalse(report.exceeds_noise("gio", 200, 224)) + self.assertTrue(report.exceeds_noise("gio", 200, 225)) + + def test_no_difference_is_no_change(self): + for implementation in ("mygo", "gio", "unknown"): + self.assertFalse(report.exceeds_noise(implementation, 100, 100)) + + +class TechnologiesTest(unittest.TestCase): + PAGE = """# Memory use + +Text with a | pipe. + +| Implementation | Technology | Processes | +| -------------------------------- | ------------------ | --------: | +| [mygo](../mygo/README.md#memory) | mygo 0.5, Go, GTK | 1 | +| [Uno](../uno/README.md#memory) | Uno Platform 6 | 1 | + +| Implementation | Other | +| --- | --- | +| [later](../later/README.md) | not read | +""" + + def test_reads_the_first_table_by_directory(self): + self.assertEqual( + report.read_technologies(self.PAGE), {"mygo": "mygo 0.5, Go, GTK", "uno": "Uno Platform 6"} + ) + + def test_a_page_without_the_table_gives_nothing(self): + self.assertEqual(report.read_technologies("# Nothing\n\n| a | b |\n| - | - |\n"), {}) + + def test_reads_the_page_of_this_repository(self): + page = Path(__file__).resolve().parents[2] / "docs" / "memory.md" + technologies = report.read_technologies(page.read_text(encoding="utf-8")) + self.assertTrue(technologies) + self.assertTrue(all(technologies.values())) + + +class RenderTest(unittest.TestCase): + def rows(self) -> list[Row]: + return [ + Row("tauri", "Tauri", "Tauri 2, Rust", "measured", memory(283, 305, 3), memory(283, 305, 3)), + Row("mygo", "mygo", "mygo 0.5, Go", "inherited", None, memory(42, 49, 1)), + Row("uno", "Uno", "", "measured", memory(101, 128), memory(101, 127)), + ] + + def render(self, rows: list[Row]) -> str: + return report.render(rows, head_sha=HEAD) + + def table(self, text: str) -> list[list[str]]: + lines = [line for line in text.splitlines() if line.startswith("|")] + return [[cell.strip() for cell in line.strip("|").split("|")] for line in lines] + + def test_starts_with_the_marker_that_finds_it_again(self): + text = self.render(self.rows()) + self.assertEqual(text.splitlines()[0], report.MARKER) + self.assertIn("`aaaaaaa`", text.splitlines()[1]) + + def test_is_sorted_by_the_pss_of_the_home_page(self): + table = self.table(self.render(self.rows())) + self.assertEqual([line[0] for line in table[2:]], ["mygo (main)", "Uno", "Tauri"]) + + def test_shows_the_difference_from_main_for_what_was_measured_only(self): + table = self.table(self.render(self.rows())) + by_name = {line[0]: line for line in table[2:]} + self.assertEqual(by_name["Tauri"][3], "283 (0)") + self.assertEqual(by_name["Uno"][5], "128 (+1)") + self.assertEqual(by_name["mygo (main)"][3], "42") + + def test_the_list_adds_what_the_versions_use_beyond_the_home_page(self): + table = self.table(self.render(self.rows())) + self.assertEqual({line[0]: line[-1] for line in table[2:]}, {"mygo (main)": "+7", "Uno": "+27", "Tauri": "+22"}) + + def test_a_change_beyond_the_noise_is_bold_and_in_the_summary(self): + rows = [Row("tauri", "Tauri", "", "measured", memory(330, 305), memory(283, 305))] + text = self.render(rows) + self.assertIn("330 (**+47**)", text) + self.assertIn("- Changed from main beyond the noise of CI: Tauri: Home PSS +47 MB (+17%).", text) + self.assertNotIn("Within the noise", text) + + def test_a_decrease_counts_too(self): + rows = [Row("tauri", "Tauri", "", "measured", memory(200, 305), memory(283, 305))] + self.assertIn("200 (**-83**)", self.render(rows)) + + def test_a_change_in_the_number_of_processes_is_a_change(self): + rows = [Row("tauri", "Tauri", "", "measured", memory(283, 305, 4), memory(283, 305, 3))] + text = self.render(rows) + self.assertIn("4 (**+1**)", text) + self.assertIn("Tauri: processes 3 -> 4", text) + + def test_unchanged_implementations_are_listed_apart_from_the_changed_one(self): + rows = [ + Row("tauri", "Tauri", "", "measured", memory(330, 305), memory(283, 305)), + Row("uno", "Uno", "", "measured", memory(101, 127), memory(101, 127)), + ] + text = self.render(rows) + self.assertIn("- Within the noise of CI: Uno.", text) + + def test_says_so_when_nothing_changed(self): + rows = [Row("uno", "Uno", "", "measured", memory(101, 127), memory(101, 127))] + self.assertIn("- No change from main beyond the noise of CI in Uno.", self.render(rows)) + + def test_an_implementation_without_numbers_on_main_has_nothing_to_compare_with(self): + rows = [Row("uno", "Uno", "", "measured", memory(101, 127), None)] + text = self.render(rows) + self.assertIn("Uno (no baseline)", text) + self.assertIn("- No numbers on main to compare with yet: Uno.", text) + self.assertIn("| 101 |", text) + + def test_failed_and_running_implementations_come_last_with_a_link(self): + rows = self.rows() + [ + Row("flet", "Flet", "", "failed", run_url="https://example.test/failed"), + Row("gio", "Gio", "", "running", run_url="https://example.test/running"), + Row("wails", "Wails", "", "missing"), + ] + text = self.render(rows) + names = [line[0] for line in self.table(text)[2:]] + self.assertEqual(names, ["mygo (main)", "Uno", "Tauri", "Gio", "Flet", "Wails"]) + self.assertIn("[no result](https://example.test/failed)", text) + self.assertIn("[running](https://example.test/running)", text) + self.assertIn("- Still running: Gio.", text) + self.assertIn("- No result, the run failed or was cancelled: Flet.", text) + self.assertIn("- No numbers anywhere: Wails.", text) + + def test_says_when_the_pull_request_rebuilt_nothing(self): + rows = [Row("mygo", "mygo", "", "inherited", None, memory(42, 49))] + self.assertIn("did not rebuild any implementation", self.render(rows)) + self.assertNotIn("did not rebuild any implementation", self.render(self.rows())) + + def test_drops_the_technology_column_without_technologies(self): + rows = [Row("uno", "Uno", "", "measured", memory(101, 127), memory(101, 127))] + table = self.table(self.render(rows)) + self.assertEqual(table[0][:3], ["Implementation", "Processes", "Home PSS"]) + self.assertEqual({len(line) for line in table}, {7}) + + def test_keeps_the_technology_column_with_technologies(self): + table = self.table(self.render(self.rows())) + self.assertEqual(table[0][:2], ["Implementation", "Technology"]) + self.assertEqual({len(line) for line in table}, {8}) + + def test_a_cell_cannot_break_the_table(self): + rows = [Row("uno", "Un|o\n", "a | b\nc", "measured", memory(101, 127), memory(101, 127))] + table = self.table(self.render(rows).replace("\\|", "/")) + self.assertEqual({len(line) for line in table}, {8}) + + def test_links_the_runs(self): + rows = [ + Row( + "uno", + "Uno", + "", + "measured", + memory(101, 127), + memory(101, 127), + run_url="https://example.test/pr", + baseline_url="https://example.test/main", + baseline_sha=MAIN, + ) + ] + text = self.render(rows) + self.assertIn("[Uno](https://example.test/pr)", text) + self.assertIn("[Uno](https://example.test/main) (`bbbbbbb`)", text) + + def test_explains_the_noise_that_it_allows(self): + rows = [ + Row("flutter", "Flutter", "", "measured", memory(300, 310), memory(300, 310)), + Row("gio", "Gio", "", "measured", memory(200, 210), memory(200, 210)), + ] + text = self.render(rows) + self.assertIn("more than 5 MB and more than 5% of the number on main", text) + self.assertIn("12% for Gio, 15% for Flutter, which vary more", text) + + def test_is_ascii(self): + self.render(self.rows()).encode("ascii") + + +class ReadArtifactTest(unittest.TestCase): + def read(self, files, **overrides): + github = FakeGitHub() + artifact_id = github.add_artifact("uno", files, **overrides) + artifact = github.artifacts[-1] + self.assertEqual(artifact["id"], artifact_id) + return report.read_artifact(github, artifact) + + def test_reads_memory_txt(self): + self.assertEqual(self.read(memory_text((55, 42, 1), (64, 49, 1))), Memory(Point(55, 42, 1), Point(64, 49, 1))) + + def test_rejects_other_files(self): + self.assertIsNone(self.read({"other.txt": memory_text()})) + self.assertIsNone(self.read({"memory.txt": memory_text(), "extra.txt": "x"})) + self.assertIsNone(self.read({"../memory.txt": memory_text()})) + + def test_rejects_an_expired_or_a_large_artifact_without_downloading_it(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text(), expired=True) + github.add_artifact("uno", memory_text(), size=report.MAX_ARTIFACT_BYTES + 1) + github.raw = mock.Mock(side_effect=AssertionError("downloaded")) + for artifact in github.artifacts: + self.assertIsNone(report.read_artifact(github, artifact)) + + def test_rejects_a_large_file_inside_the_zip(self): + self.assertIsNone(self.read(memory_text() + "x" * (report.MAX_MEMORY_FILE_BYTES + 1))) + + def test_rejects_what_is_not_a_zip(self): + github = FakeGitHub() + artifact_id = github.add_artifact("uno", memory_text()) + github.zips[artifact_id] = b"not a zip" + with contextlib.redirect_stderr(io.StringIO()): + self.assertIsNone(report.read_artifact(github, github.artifacts[-1])) + + def test_rejects_wrong_numbers(self): + self.assertIsNone(self.read("memory (home): RSS many MB, PSS 1 MB in 1 processes\n")) + + +class BaselineTest(unittest.TestCase): + def test_takes_the_newest_valid_artifact_of_the_default_branch(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text((10, 9, 1), (10, 9, 1)), created="2026-10-18T00:00:00Z", sha="1" * 40) + github.add_artifact("uno", memory_text((20, 19, 1), (20, 19, 1)), created="2026-10-20T00:00:00Z", sha="2" * 40) + baseline, artifact = report.baseline_of(github, "uno", "main") + self.assertEqual(baseline.home.pss, 19) + self.assertEqual(artifact["workflow_run"]["head_sha"], "2" * 40) + + def test_ignores_pull_requests_forks_and_expired_artifacts(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text(), branch="feature", created="2026-10-22T00:00:00Z") + github.add_artifact("uno", memory_text(), head_repository_id=2, created="2026-10-22T00:00:00Z") + github.add_artifact("uno", memory_text(), expired=True, created="2026-10-22T00:00:00Z") + self.assertIsNone(report.baseline_of(github, "uno", "main")) + + def test_falls_back_to_an_older_artifact_when_the_newest_is_invalid(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text((10, 9, 1), (10, 9, 1)), created="2026-10-18T00:00:00Z") + github.add_artifact("uno", "garbage", created="2026-10-20T00:00:00Z") + self.assertEqual(report.baseline_of(github, "uno", "main")[0].home.pss, 9) + + def test_only_reads_the_artifacts_of_the_implementation(self): + github = FakeGitHub() + github.add_artifact("tauri", memory_text()) + self.assertIsNone(report.baseline_of(github, "uno", "main")) + + def test_ignores_an_artifact_that_no_workflow_run_made(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text()) + github.artifacts[-1]["workflow_run"] = None + self.assertIsNone(report.baseline_of(github, "uno", "main")) + + +class PullRequestTest(unittest.TestCase): + def test_finds_it_by_the_branch_of_the_run(self): + github = FakeGitHub() + github.add_pull_request(7) + pulls = report.find_pull_requests(github, "contributor", "feature", HEAD) + self.assertEqual([pull["number"] for pull in pulls], [7]) + + def test_finds_it_by_the_commit_without_the_owner(self): + github = FakeGitHub() + github.add_pull_request(7) + self.assertEqual([pull["number"] for pull in report.find_pull_requests(github, None, "feature", HEAD)], [7]) + + def test_a_pull_request_that_has_a_newer_commit_is_not_the_one_of_the_run(self): + github = FakeGitHub() + github.add_pull_request(7, sha="c" * 40) + self.assertEqual(report.find_pull_requests(github, "contributor", "feature", HEAD), []) + + def test_a_closed_pull_request_is_not_found(self): + github = FakeGitHub() + github.add_pull_request(7, state="closed") + self.assertEqual(report.find_pull_requests(github, "contributor", "feature", HEAD), []) + + +class UpsertCommentTest(unittest.TestCase): + def comment(self, github, body, login=report.BOT, number=7): + comment = {"id": github.next_id(), "body": body, "user": {"login": login}} + github.comments.setdefault(number, []).append(comment) + return comment + + def test_creates_the_first_comment(self): + github = FakeGitHub() + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nnew"), "created") + self.assertEqual(github.writes, [("POST", "issues/7/comments")]) + + def test_replaces_the_earlier_comment_instead_of_adding_another(self): + github = FakeGitHub() + old = self.comment(github, f"{report.MARKER}\nold") + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nnew"), "updated") + self.assertEqual(github.writes, [("PATCH", f"issues/comments/{old['id']}")]) + self.assertEqual([c["body"] for c in github.comments[7]], [f"{report.MARKER}\nnew"]) + + def test_leaves_an_identical_comment_alone(self): + github = FakeGitHub() + self.comment(github, f"{report.MARKER}\r\nsame\r\n") + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nsame\n"), "unchanged") + self.assertEqual(github.writes, []) + + def test_keeps_the_oldest_comment_and_deletes_the_rest(self): + github = FakeGitHub() + first = self.comment(github, f"{report.MARKER}\nfirst") + second = self.comment(github, f"{report.MARKER}\nsecond") + report.upsert_comment(github, 7, f"{report.MARKER}\nnew") + self.assertEqual( + github.writes, [("PATCH", f"issues/comments/{first['id']}"), ("DELETE", f"issues/comments/{second['id']}")] + ) + self.assertEqual([c["body"] for c in github.comments[7]], [f"{report.MARKER}\nnew"]) + + def test_never_touches_the_comments_of_other_people(self): + github = FakeGitHub() + theirs = self.comment(github, f"{report.MARKER}\nquoted by a person", login="contributor") + other = self.comment(github, "a remark by the bot") + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nnew"), "created") + self.assertEqual(github.writes, [("POST", "issues/7/comments")]) + self.assertIn(theirs, github.comments[7]) + self.assertIn(other, github.comments[7]) + + def test_tolerates_a_comment_of_a_deleted_user(self): + github = FakeGitHub() + github.comments[7] = [{"id": 1, "body": "text", "user": None}] + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nnew"), "created") + + +class ReportTest(unittest.TestCase): + """The whole report, against a fake GitHub.""" + + # The numbers on main: (name of the workflow, Home as RSS, PSS, processes, Versions as RSS, PSS, processes). + MAIN_NUMBERS = { + "mygo": ("mygo", (55, 42, 1), (64, 49, 1)), + "tauri": ("Tauri", (470, 283, 3), (492, 305, 3)), + "uno": ("Uno", (110, 101, 1), (136, 127, 1)), + } + + def setUp(self): + self.github = FakeGitHub() + for implementation, (name, home, versions) in self.MAIN_NUMBERS.items(): + self.github.add_workflow(implementation, name) + self.github.add_artifact(implementation, memory_text(home, versions)) + self.github.add_pull_request(7) + + def run_report(self, **overrides): + arguments = dict( + implementations=["mygo", "tauri", "uno"], + technologies={"mygo": "mygo 0.5, Go"}, + run_id=None, + pull_request=7, + dry_run=False, + ) + arguments.update(overrides) + output, errors = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + status = report.report(self.github, **arguments) + return status, output.getvalue(), errors.getvalue() + + def body(self) -> str: + (comment,) = self.github.comments[7] + return comment["body"] + + def test_comments_with_the_measured_implementation_and_main_for_the_rest(self): + run = self.github.add_run("mygo", memory_text((56, 43, 1), (65, 50, 1))) + status, _, _ = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + table = [line for line in self.body().splitlines() if line.startswith("|")] + self.assertEqual([line.split("|")[1].strip() for line in table[2:]], ["mygo", "Uno (main)", "Tauri (main)"]) + self.assertIn("43 (+1)", self.body()) + self.assertIn("No change from main beyond the noise of CI in mygo.", self.body()) + + def test_waits_for_the_other_implementations_that_are_still_running(self): + run = self.github.add_run("mygo", memory_text()) + self.github.add_run("tauri", status="in_progress", conclusion=None) + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + self.assertIn("Waiting for Tauri", errors) + self.assertEqual(self.github.comments, {}) + + def test_waiting_takes_few_requests(self): + # Every workflow that ends starts a report, and all but the last one only wait. + run = self.github.add_run("mygo", memory_text()) + self.github.add_run("tauri", status="in_progress", conclusion=None) + paths = [] + original = self.github.get + + def get(path, **params): + paths.append(path) + return original(path, **params) + + self.github.get = get + self.github.raw = mock.Mock(side_effect=AssertionError("downloaded an artifact")) + self.run_report(run_id=run, pull_request=None) + self.assertLessEqual(len(paths), 6, paths) + self.assertFalse([path for path in paths if "artifacts" in path], paths) + + def test_the_last_run_to_end_writes_the_report_with_all_of_them(self): + self.github.add_run("tauri", memory_text((470, 283, 3), (492, 305, 3))) + run = self.github.add_run("mygo", memory_text()) + self.run_report(run_id=run, pull_request=None) + table = [line for line in self.body().splitlines() if line.startswith("|")] + self.assertEqual([line.split("|")[1].strip() for line in table[2:]], ["mygo", "Uno (main)", "Tauri"]) + + def test_the_run_that_ended_counts_as_ended_even_if_the_list_is_late(self): + run = self.github.add_run("mygo", memory_text(), status="in_progress", conclusion=None) + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertNotIn("Waiting", errors) + self.assertIn("mygo", self.body()) + + def test_a_failed_run_is_reported_as_such(self): + self.github.add_run("tauri", conclusion="failure") + run = self.github.add_run("mygo", memory_text()) + self.run_report(run_id=run, pull_request=None) + self.assertIn("No result, the run failed or was cancelled: Tauri.", self.body()) + + def test_an_invalid_artifact_is_reported_as_no_result(self): + run = self.github.add_run("mygo", "not the numbers") + self.run_report(run_id=run, pull_request=None) + self.assertIn("No result, the run failed or was cancelled: mygo.", self.body()) + + def test_the_second_report_replaces_the_comment_of_the_first(self): + run = self.github.add_run("mygo", memory_text((56, 43, 1), (65, 50, 1))) + self.run_report(run_id=run, pull_request=None) + self.github.pulls[7]["head"]["sha"] = "c" * 40 + later = self.github.add_run("mygo", memory_text((60, 47, 1), (70, 54, 1)), sha="c" * 40) + self.run_report(run_id=later, pull_request=None) + self.assertEqual(len(self.github.comments[7]), 1) + self.assertIn("47 (+5)", self.body()) + self.assertIn("`ccccccc`", self.body()) + self.assertEqual([method for method, _ in self.github.writes], ["POST", "PATCH"]) + + def test_ignores_the_run_of_a_commit_that_is_no_longer_the_latest(self): + run = self.github.add_run("mygo", memory_text()) + self.github.pulls[7]["head"]["sha"] = "c" * 40 + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + self.assertIn("No open pull request has aaaaaaa", errors) + self.assertEqual(self.github.writes, []) + + def test_does_not_comment_when_the_commit_changes_during_the_report(self): + run = self.github.add_run("mygo", memory_text()) + original = self.github.get + + def get(path, **params): + # The first read of the pull request is the one that finds it; later reads see a push. + if path == "pulls/7": + self.github.pulls[7]["head"]["sha"] = "c" * 40 + return original(path, **params) + + self.github.get = get + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + self.assertIn("has moved on", errors) + self.assertEqual(self.github.writes, []) + + def test_ignores_runs_that_are_not_of_a_pull_request(self): + run = self.github.add_run("mygo", memory_text(), event="push", sha=MAIN) + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + self.assertIn("not one of a pull request", errors) + self.assertEqual(self.github.writes, []) + + def test_skips_a_workflow_that_no_longer_exists(self): + self.github.workflows[1]["state"] = "deleted" + self.run_report() + self.assertNotIn("Tauri", self.body()) + + def test_an_implementation_without_a_workflow_is_skipped(self): + status, _, errors = self.run_report(implementations=["nothing"]) + self.assertEqual(status, 0) + self.assertIn("None of the implementations has a workflow", errors) + + def test_manual_mode_does_not_wait(self): + self.github.add_run("mygo", memory_text()) + self.github.add_run("tauri", status="in_progress", conclusion=None) + self.run_report() + self.assertIn("Still running: Tauri.", self.body()) + + def test_manual_mode_on_a_closed_pull_request_does_nothing(self): + self.github.pulls[7]["state"] = "closed" + status, _, errors = self.run_report() + self.assertEqual(status, 0) + self.assertIn("is not open", errors) + + def test_dry_run_prints_and_writes_nothing(self): + self.github.add_run("mygo", memory_text()) + _, output, _ = self.run_report(dry_run=True) + self.assertTrue(output.startswith(report.MARKER)) + self.assertEqual(self.github.writes, []) + + def test_a_new_implementation_has_no_baseline_and_is_still_reported(self): + self.github.artifacts = [a for a in self.github.artifacts if a["name"] != "memory-uno"] + run = self.github.add_run("uno", memory_text()) + self.run_report(run_id=run, pull_request=None) + self.assertIn("Uno (no baseline)", self.body()) + + def test_writes_the_step_summary(self): + self.github.add_run("mygo", memory_text()) + with tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / "summary.md" + with mock.patch.dict("os.environ", {"GITHUB_STEP_SUMMARY": str(summary)}): + self.run_report() + text = summary.read_text(encoding="utf-8") + self.assertTrue(text.startswith("### Memory use")) + self.assertNotIn(report.MARKER, text) + + +class GitHubTest(unittest.TestCase): + def completed(self, returncode=0, stdout=b"{}", stderr=b""): + return subprocess.CompletedProcess([], returncode, stdout, stderr) + + def test_builds_the_endpoint_of_the_repository(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed()) as run: + github.get("actions/runs", head_sha="abc", per_page=100) + endpoint = f"repos/{REPOSITORY}/actions/runs?head_sha=abc&per_page=100" + self.assertEqual(run.call_args.args[0], ["gh", "api", "--method", "GET", endpoint]) + + def test_retries_a_failure_of_the_server(self): + github = report.GitHub(REPOSITORY) + answers = [self.completed(1, b"", b"gh: HTTP 502"), self.completed(0, b'{"a": 1}')] + with mock.patch("subprocess.run", side_effect=answers), mock.patch("time.sleep") as sleep: + self.assertEqual(github.get("x"), {"a": 1}) + sleep.assert_called_once() + + def test_does_not_retry_a_refusal(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed(1, b"", b"gh: Not Found (HTTP 404)")) as run: + with self.assertRaises(report.ApiError): + github.get("x") + self.assertEqual(run.call_count, 1) + + def test_gives_up_after_three_attempts(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed(1, b"", b"gh: HTTP 503")) as run: + with mock.patch("time.sleep"), self.assertRaises(report.ApiError): + github.get("x") + self.assertEqual(run.call_count, 3) + + def test_creating_a_comment_is_tried_once(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed(1, b"", b"gh: HTTP 502")) as run: + with mock.patch("time.sleep"), self.assertRaises(report.ApiError): + github.send("POST", "issues/7/comments", {"body": "x"}) + self.assertEqual(run.call_count, 1) + + def test_sends_the_body_on_standard_input_and_accepts_an_empty_answer(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed(0, b"")) as run: + self.assertIsNone(github.send("DELETE", "issues/comments/1")) + github.send("PATCH", "issues/comments/1", {"body": "text"}) + self.assertEqual(run.call_args.kwargs["input"], b'{"body": "text"}') + self.assertEqual(run.call_args.args[0][-2:], ["--input", "-"]) + + def test_reads_every_page(self): + github = report.GitHub(REPOSITORY) + pages = [list(range(100)), list(range(5))] + with mock.patch.object(report.GitHub, "get", side_effect=pages): + self.assertEqual(len(list(github.pages("issues/7/comments"))), 105) + + +class MainTest(unittest.TestCase): + def run_main(self, *arguments): + with contextlib.redirect_stderr(io.StringIO()): + return report.main(list(arguments)) + + def test_rejects_a_repository_that_is_not_owner_and_name(self): + with self.assertRaises(SystemExit): + self.run_main("--repository", "x;y", "--implementations", "mygo", "--pull-request", "1") + + def test_rejects_an_implementation_that_is_not_a_plain_name(self): + with self.assertRaises(SystemExit): + self.run_main("--repository", REPOSITORY, "--implementations", "../mygo", "--pull-request", "1") + with self.assertRaises(SystemExit): + self.run_main("--repository", REPOSITORY, "--implementations", "", "--pull-request", "1") + + def test_needs_a_run_or_a_pull_request(self): + with self.assertRaises(SystemExit): + self.run_main("--repository", REPOSITORY, "--implementations", "mygo") + + def test_reports_a_failure_of_the_api_as_an_exit_code(self): + with mock.patch.object(report, "report", side_effect=report.ApiError("boom")): + status = self.run_main( + "--repository", REPOSITORY, "--implementations", "mygo", "--pull-request", "1", "--docs", "missing.md" + ) + self.assertEqual(status, 1) + + def test_the_warning_is_one_line_of_a_workflow_command(self): + error = report.ApiError("POST x: gh: Resource not accessible by integration (HTTP 403)\nsecond line 100%") + output = io.StringIO() + with mock.patch.object(report, "report", side_effect=error), contextlib.redirect_stdout(output): + self.run_main( + "--repository", REPOSITORY, "--implementations", "mygo", "--pull-request", "1", "--docs", "missing.md" + ) + self.assertEqual(len(output.getvalue().splitlines()), 1) + self.assertIn("%0Asecond line 100%25", output.getvalue()) + + def test_a_token_that_cannot_write_is_a_warning_and_not_a_failure(self): + error = report.ApiError("POST issues/7/comments: gh: Resource not accessible by integration (HTTP 403)") + output = io.StringIO() + with mock.patch.object(report, "report", side_effect=error), contextlib.redirect_stdout(output): + status = self.run_main( + "--repository", REPOSITORY, "--implementations", "mygo", "--pull-request", "1", "--docs", "missing.md" + ) + self.assertEqual(status, 0) + self.assertTrue(output.getvalue().startswith("::warning::")) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/capacitor.yml b/.github/workflows/capacitor.yml index 7cfd998..a802fe0 100644 --- a/.github/workflows/capacitor.yml +++ b/.github/workflows/capacitor.yml @@ -118,11 +118,20 @@ jobs: - name: Build the end-to-end test build run: npm run build:e2e - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash ../web/e2e/run.sh "$scenario" "$PWD/node_modules/electron/dist/electron" "$PWD/dist/app-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-capacitor + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Build the release run: | npm run build diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index b596269..b676ec7 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -107,6 +107,9 @@ jobs: - name: Build the application image run: gradle --console=plain createDistributable - name: Smoke test the application image and measure its memory + env: + # The measurement writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | binary="$PWD/build/compose/binaries/main/app/vfox-plus/bin/vfox-plus" xvfb-run -a -s "-screen 0 1280x800x24" "$binary" > app.log 2>&1 & @@ -118,6 +121,12 @@ jobs: fi pkill -f "^$binary" xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- bash ../shared/tools/measure.sh "$binary" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-compose + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Build the .deb and the tarball run: | # Reproducible timestamps in the package metadata. diff --git a/.github/workflows/dioxus.yml b/.github/workflows/dioxus.yml index da28098..e1e5554 100644 --- a/.github/workflows/dioxus.yml +++ b/.github/workflows/dioxus.yml @@ -125,11 +125,20 @@ jobs: cargo build --release --locked --features e2e cp target/release/vfox-plus "$RUNNER_TEMP/vfox-plus-e2e" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-dioxus + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Build the release working-directory: dioxus run: cargo build --release --locked diff --git a/.github/workflows/flet.yml b/.github/workflows/flet.yml index 9bd337c..35c7686 100644 --- a/.github/workflows/flet.yml +++ b/.github/workflows/flet.yml @@ -126,12 +126,21 @@ jobs: done - name: Run the end-to-end scenarios in a virtual display working-directory: . + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ env VFOX_PLUS_E2E_TIMEOUT=300 VFOX_PLUS_E2E_VFOX="$VFOX_PLUS_E2E_VFOX" \ bash web/e2e/run.sh "$scenario" "$PWD/flet/build/linux/vfox-plus" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-flet + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Smoke test the release run: | binary="$PWD/build/linux/vfox-plus" diff --git a/.github/workflows/flutter.yml b/.github/workflows/flutter.yml index 835416f..64ff8f0 100644 --- a/.github/workflows/flutter.yml +++ b/.github/workflows/flutter.yml @@ -193,6 +193,13 @@ jobs: run: | xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ python3 tool/measure.py --output "$RUNNER_TEMP/measure" build/linux/x64/release/bundle/vfox-plus + # The numbers, for memory.yml. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-flutter + path: ${{ runner.temp }}/measure/memory.txt + if-no-files-found: error + overwrite: true - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: failure() with: diff --git a/.github/workflows/gio.yml b/.github/workflows/gio.yml index 03c4199..8307b4a 100644 --- a/.github/workflows/gio.yml +++ b/.github/workflows/gio.yml @@ -122,12 +122,21 @@ jobs: tar -xzf "$RUNNER_TEMP/$archive.tar.gz" -C "$RUNNER_TEMP" echo "VFOX_PLUS_E2E_VFOX=$RUNNER_TEMP/$archive/vfox" >> "$GITHUB_ENV" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | bash gio/tools/build.sh --e2e "$RUNNER_TEMP/vfox-plus-e2e" for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-gio + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Smoke test the release run: | binary="$PWD/gio/build/vfox-plus" diff --git a/.github/workflows/memory.yml b/.github/workflows/memory.yml new file mode 100644 index 0000000..3a73ac9 --- /dev/null +++ b/.github/workflows/memory.yml @@ -0,0 +1,86 @@ +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The memory of the implementations in a pull request. The workflow of each +# implementation measures its application and uploads the result as the +# artifact memory-. When the last of those workflows has ended +# for the latest commit of a pull request, this one writes a single comment on +# it: a table like the one of docs/memory.md with the difference from the latest +# numbers of main. Later runs replace that comment instead of adding another one +# (.github/scripts/memory_report.py; docs/memory.md#memory-in-pull-requests). +# +# It runs after the workflows of the pull request, in the context of the default +# branch, so that it can write to pull requests of forks, which a pull_request +# workflow cannot. For the same reason it never runs code of the pull request: it +# checks out main, and the artifacts it downloads are data, of which the script +# only reads numbers. +# +# Run it by hand to write the report of a pull request again: +# gh workflow run memory.yml -f pull_request= +name: Memory + +on: + workflow_run: + # The workflows that measure an implementation, by name. Keep this in sync + # with MEMORY_IMPLEMENTATIONS below, which has their file names. + workflows: [Capacitor, Compose, Dioxus, Flet, Flutter, Gio, mygo, Tauri, Uno, Wails] + types: [completed] + workflow_dispatch: + inputs: + pull_request: + description: The number of the pull request to write the report of now + required: true + type: number + +permissions: + actions: read + contents: read + pull-requests: write + +concurrency: + # The workflows of a commit end one after another, and each starts a report. + # They run one at a time, so that two of them cannot both write the comment, + # and the last one to start is the one that finds all of them ended. + group: memory-${{ github.event.workflow_run.head_sha || format('pull-request-{0}', inputs.pull_request) }} + cancel-in-progress: false + +env: + # The file names of the workflows of the implementations (without .yml), which + # are also the names of their directories and of their artifacts. + MEMORY_IMPLEMENTATIONS: capacitor compose dioxus flet flutter gio mygo tauri uno wails + +jobs: + report: + name: Write the report + # Forks do not run CI. + if: >- + github.repository == 'version-fox/vfox-plus' && + (github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'pull_request') + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Write the report + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + PULL_REQUEST: ${{ inputs.pull_request }} + run: | + if [[ -n "$RUN_ID" ]]; then + python3 .github/scripts/memory_report.py --run "$RUN_ID" + else + python3 .github/scripts/memory_report.py --pull-request "$PULL_REQUEST" + fi diff --git a/.github/workflows/mygo.yml b/.github/workflows/mygo.yml index fc2452e..0d1044b 100644 --- a/.github/workflows/mygo.yml +++ b/.github/workflows/mygo.yml @@ -122,12 +122,21 @@ jobs: tar -xzf "$RUNNER_TEMP/$archive.tar.gz" -C "$RUNNER_TEMP" echo "VFOX_PLUS_E2E_VFOX=$RUNNER_TEMP/$archive/vfox" >> "$GITHUB_ENV" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | bash mygo/tools/build.sh --e2e "$RUNNER_TEMP/vfox-plus-e2e" for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-mygo + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Smoke test the release run: | binary="$PWD/mygo/build/vfox-plus" diff --git a/.github/workflows/repository.yml b/.github/workflows/repository.yml index b93993f..165034b 100644 --- a/.github/workflows/repository.yml +++ b/.github/workflows/repository.yml @@ -78,3 +78,6 @@ jobs: for file in shared/icons.json shared/i18n/*.arb shared/fixtures/*/cases.json; do jq empty "$file" || { echo "::error file=$file::Invalid JSON"; exit 1; } done + - name: Test the memory report + # What memory.yml writes on pull requests. + run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' diff --git a/.github/workflows/tauri.yml b/.github/workflows/tauri.yml index e0f746f..601ec06 100644 --- a/.github/workflows/tauri.yml +++ b/.github/workflows/tauri.yml @@ -115,11 +115,20 @@ jobs: '{"build": {"frontendDist": "../../web/dist/tauri-e2e", "beforeBuildCommand": ""}}' cp src-tauri/target/release/vfox-plus "$RUNNER_TEMP/vfox-plus-e2e" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-tauri + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Build the release working-directory: tauri run: npx tauri build --no-bundle diff --git a/.github/workflows/uno.yml b/.github/workflows/uno.yml index ed41f91..551e471 100644 --- a/.github/workflows/uno.yml +++ b/.github/workflows/uno.yml @@ -137,12 +137,21 @@ jobs: working-directory: uno run: dotnet restore src/VfoxPlus.App --runtime linux-x64 -p:Configuration=Release - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | bash uno/tools/publish.sh --e2e "$RUNNER_TEMP/vfox-plus-e2e" for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e/VfoxPlus" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-uno + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Publish the release run: bash uno/tools/publish.sh uno/build/stage - name: Build the .deb and the tarball diff --git a/.github/workflows/wails.yml b/.github/workflows/wails.yml index 7baebe4..319a371 100644 --- a/.github/workflows/wails.yml +++ b/.github/workflows/wails.yml @@ -120,12 +120,21 @@ jobs: tar -xzf "$RUNNER_TEMP/$archive.tar.gz" -C "$RUNNER_TEMP" echo "VFOX_PLUS_E2E_VFOX=$RUNNER_TEMP/$archive/vfox" >> "$GITHUB_ENV" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | bash wails/build.sh --e2e "$RUNNER_TEMP/vfox-plus-e2e" for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-wails + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Smoke test the release run: | binary="$PWD/wails/build/vfox-plus" diff --git a/.gitignore b/.gitignore index 2c32fd6..d6ae8b1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,8 @@ Thumbs.db # Packaging output (see flutter/packaging/linux/build_deb.sh). /dist/ + +# Python bytecode, which running the tests of .github/scripts leaves behind. CI +# rejects binary files. +__pycache__/ +*.pyc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b758eb..f71bfe3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,8 @@ all of them. | Workflow | Runs on | Checks | | --- | --- | --- | -| [`repository.yml`](.github/workflows/repository.yml) | every change | License headers (SkyWalking Eyes), workflows (actionlint), shell scripts (ShellCheck), shared JSON, no committed binary files | +| [`repository.yml`](.github/workflows/repository.yml) | every change | License headers (SkyWalking Eyes), workflows (actionlint), shell scripts (ShellCheck), shared JSON, no committed binary files, the tests of the memory report | +| [`memory.yml`](.github/workflows/memory.yml) | the end of the workflows of the implementations, on pull requests | One comment on the pull request with the memory of every implementation, sorted like [docs/memory.md](docs/memory.md#results), and the difference from `main` (see [Memory in pull requests](docs/memory.md#memory-in-pull-requests)) | | [`flutter.yml`](.github/workflows/flutter.yml) | `flutter/`, `shared/` | Lock file, formatting, generated translations, analysis, tests with an 85 % coverage gate, end-to-end tests on Linux under Xvfb against captured output and against the real vfox, release build with a smoke test and a memory measurement, `.deb` and `.tar.gz` packages validated with lintian | | [`flutter-sandbox.yml`](.github/workflows/flutter-sandbox.yml) | `flutter/sandbox/`, `flutter/versions.env` | Builds the Docker sandbox and runs `sandbox.sh all` in it | | [`web.yml`](.github/workflows/web.yml) | `web/`, `shared/` | Formatting (Prettier), lint (ESLint), type checking, unit tests with an 85 % coverage gate, the production builds | @@ -71,6 +72,12 @@ all of them. Dependabot keeps the pub, npm, cargo, Go, Gradle and NuGet dependencies and the GitHub Actions up to date; actions are pinned to commit SHAs. +A pull request that rebuilds an implementation gets a comment with the memory of all of +them and the difference from `main`, which the next commit replaces instead of adding +another one. A change that is more than the noise between CI runs is in bold; read +[Memory in pull requests](docs/memory.md#memory-in-pull-requests) before you trust a small +one. + ## Common changes ### Texts diff --git a/docs/memory.md b/docs/memory.md index 27656fb..7f0d975 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -146,6 +146,78 @@ What is different from the other implementations: - The memory is read when the screen has been still for 3 seconds, not when the application reports a page. +## Memory in pull requests + +A pull request that rebuilds an implementation gets one comment with the memory of all +of them: a table like [the one above](#results), sorted by the PSS of the Home page, +with the difference from `main` next to the PSS. A new commit replaces that comment +instead of adding another one. + +- The workflow of each implementation runs the `measure` scenario of [How it is + measured](#how-it-is-measured) (Flutter: `tool/measure.py`) and uploads the lines that + it prints as the artifact `memory-`. `measure.sh` writes them to the file + named in `VFOX_PLUS_MEASURE_OUTPUT` as well. +- [`memory.yml`](../.github/workflows/memory.yml) starts every time one of those workflows + ends and runs [`memory_report.py`](../.github/scripts/memory_report.py). While another + one is still running for the latest commit of the pull request, the script stops. The + run after the last one compares the artifacts of the pull request with those of the + latest run of the same workflow on `main` and writes the comment. The workflow runs in + the context of `main`, which is what lets it comment on pull requests of forks, and it + reads nothing but two lines of numbers from the artifacts. If its token cannot write the + comment, the run only warns, and the report is in its summary. +- An implementation that the pull request did not rebuild, because its workflow only runs + for changes in its own paths, shows the numbers of `main` and is marked "(main)". +- The numbers are not those of the table above. They come from the runners of GitHub and + from one run, not from Docker on a WSL2 host. Compare the rows of one comment with each + other and with `main`, not with this page. + +### The noise + +The same code does not use the same memory in every CI run. This is the largest +difference between runs of identical code, from the 107 successful runs of 2026-10-18 and +19 whose source was the same as that of at least one other run (30 groups of 2 to 8 runs), +in MB and as a part of the lowest number: + +| Implementation | Home PSS | Versions PSS | +| -------------- | ---------: | -----------: | +| mygo | 1 (2.4%) | 0 (0.0%) | +| Uno | 4 (3.3%) | 5 (3.0%) | +| Gio | 20 (10.4%) | 23 (11.4%) | +| Dioxus | 5 (1.8%) | 13 (4.3%) | +| Compose | 15 (6.1%) | 14 (5.4%) | +| Tauri | 14 (3.3%) | 14 (2.9%) | +| Capacitor | 4 (1.0%) | 3 (0.7%) | +| Flutter | 35 (10.8%) | 43 (13.4%) | +| Flet | 16 (5.2%) | 25 (7.5%) | +| Wails | 11 (2.3%) | 13 (2.4%) | + +So the comment calls a difference a change, and prints it in bold, only when it is more +than 5 MB and more than 5% of the number on `main`; for Compose and Flet more than 8%, for +Gio more than 12% and for Flutter more than 15%, because they vary more. Of the 940 +comparisons of two runs of the same code (the Home and the Versions PSS of every pair, in +both directions), none was a change by these limits. The number of processes never varied, +so a different number is a change too. The sample is small: if the comment reports a +change that the commit cannot have caused, measure how much identical code varies again +and adjust `TOLERANCES` in the script. + +### Using it + +- The artifacts expire after 90 days. An implementation without a run on `main` for that + long has "no baseline" until its workflow runs on `main` again. +- `gh workflow run memory.yml -f pull_request=` writes the report of an open pull + request again. To only see it, run the script with `--dry-run`: + + ```sh + GH_TOKEN="$(gh auth token)" python3 .github/scripts/memory_report.py --dry-run \ + --repository version-fox/vfox-plus --pull-request \ + --implementations "capacitor compose dioxus flet flutter gio mygo tauri uno wails" + ``` + +- The tests of the script run in the `Repository` workflow: + `python3 -m unittest discover -s .github/scripts`. +- An implementation that is added uploads its numbers as `memory-` like the + others and goes into the two lists of `memory.yml`; one that is removed leaves them. + ## Updating this page When an implementation is added, or its memory changes noticeably, run diff --git a/docs/new-implementation.md b/docs/new-implementation.md index 49fa19b..f197675 100644 --- a/docs/new-implementation.md +++ b/docs/new-implementation.md @@ -68,6 +68,13 @@ state layer, and the UI on top. the directory, `shared/` and the workflow itself, and that builds the installable packages as artifacts. Use [`flutter.yml`](../.github/workflows/flutter.yml) as a model. +- In that workflow, the memory measurement (see below), uploaded as the artifact + `memory-` right after it, and the name of the workflow and + `` in the two lists of [`memory.yml`](../.github/workflows/memory.yml), + so that pull requests get the memory of the implementation in a comment (see + [the memory page](memory.md#memory-in-pull-requests)). `VFOX_PLUS_MEASURE_OUTPUT` + makes `measure.sh` write the numbers to the file that the artifact holds; copy the + steps of [`mygo.yml`](../.github/workflows/mygo.yml). ## 5. Package diff --git a/shared/tools/measure.sh b/shared/tools/measure.sh index 5ac0c88..e361b71 100755 --- a/shared/tools/measure.sh +++ b/shared/tools/measure.sh @@ -28,6 +28,10 @@ # "e2e: scenario measure passed" and exit with code 0, waiting a few seconds # after each point so that the measurement sees a settled application. Prints # lines "memory (): RSS MB, PSS MB in processes". +# +# With VFOX_PLUS_MEASURE_OUTPUT set to a file name, the same lines also replace +# the content of that file. CI uploads it for the memory report of pull +# requests (.github/workflows/memory.yml). set -euo pipefail TOOLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -51,6 +55,12 @@ cleanup() { } trap cleanup EXIT +OUTPUT="${VFOX_PLUS_MEASURE_OUTPUT:-}" +if [[ -n "$OUTPUT" ]]; then + mkdir -p "$(dirname "$OUTPUT")" + : > "$OUTPUT" +fi + mkdir -p "$HOME_DIR/.config/vfox-plus" vfox="$SANDBOX/vfox" install -m 755 "$SHARED/fixtures/fake-vfox.sh" "$vfox" @@ -85,13 +95,15 @@ APP_PID=$! printed=0 deadline=$((SECONDS + ${VFOX_PLUS_E2E_TIMEOUT:-300})) show_new_lines() { - local lines + local lines result lines="$(wc -l < "$SANDBOX/app.log")" if [[ $lines -gt $printed ]]; then while IFS= read -r line; do echo "$line" if [[ "$line" == *"e2e: MEASURE "* ]]; then - echo "memory (${line##*MEASURE }): $(memory_of "$APP_PID")" + result="memory (${line##*MEASURE }): $(memory_of "$APP_PID")" + echo "$result" + [[ -z "$OUTPUT" ]] || echo "$result" >> "$OUTPUT" fi done < <(sed -n "$((printed + 1)),${lines}p" "$SANDBOX/app.log") printed=$lines From 4e5a1cabf5a44a2c3b7568d63f2fad10d76c59f1 Mon Sep 17 00:00:00 2001 From: Qiheng He Date: Fri, 23 Oct 2026 14:46:35 +0800 Subject: [PATCH 2/2] Rename a variable that CodeQL took for a secret py/clear-text-logging-sensitive-data treats every name that contains "trusted" as secret data. It flagged the artifact name and ID that read_artifact() logs when it cannot read an artifact (memory_report.py, line 418), through the list that baseline_of() called "trusted". Nothing secret is logged: the list only held the artifacts that were made on the default branch. --- .github/scripts/memory_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/memory_report.py b/.github/scripts/memory_report.py index cbb48ef..5e9e1e1 100644 --- a/.github/scripts/memory_report.py +++ b/.github/scripts/memory_report.py @@ -447,13 +447,13 @@ def baseline_of(gh: GitHub, implementation: str, default_branch: str) -> tuple[M """The latest numbers of an implementation that were measured on the default branch.""" artifacts = gh.get("actions/artifacts", name=f"memory-{implementation}", per_page=100)["artifacts"] # A pull request from a fork can have a branch called like the default branch. - trusted = [ + on_default_branch = [ artifact for artifact in artifacts if (artifact.get("workflow_run") or {}).get("head_branch") == default_branch and artifact["workflow_run"].get("head_repository_id") == artifact["workflow_run"].get("repository_id") ] - for artifact in sorted(trusted, key=lambda candidate: candidate["created_at"], reverse=True): + for artifact in sorted(on_default_branch, key=lambda candidate: candidate["created_at"], reverse=True): memory = read_artifact(gh, artifact) if memory is not None: return memory, artifact