diff --git a/.github/scripts/memory_report.py b/.github/scripts/memory_report.py new file mode 100644 index 0000000..5e9e1e1 --- /dev/null +++ b/.github/scripts/memory_report.py @@ -0,0 +1,686 @@ +#!/usr/bin/env python3 +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Comments on a pull request with the memory that every implementation uses. + +The workflow of each implementation (.github/workflows/.yml) +measures its application with shared/tools/measure.sh (Flutter: +flutter/tool/measure.py) and uploads the result as the artifact +memory-, which holds one file, memory.txt, with the lines +"memory (): RSS MB, PSS MB in processes" of the points home +and nodejs. .github/workflows/memory.yml runs this script when one of those +workflows ends. Once all of them have ended for the latest commit of the pull +request, the script writes one comment: a table like the one of docs/memory.md, +sorted by the PSS of the Home page, with the difference from the latest +measurement on the default branch next to the numbers. A later run replaces +that comment instead of adding another one. + +An implementation that the pull request did not rebuild, because its workflow +only runs for changes in its own paths, shows the numbers of the default branch. + +Usage: + + memory_report.py --run RUN_ID [--dry-run] + Reports on the pull request of the workflow run RUN_ID, if that was the + last of the implementation workflows to end for the commit. + + memory_report.py --pull-request NUMBER [--dry-run] + Reports on that pull request now, with the implementations that are still + running marked as such. + +--dry-run prints the comment instead of writing it. The implementations come +from --implementations or MEMORY_IMPLEMENTATIONS (names separated by spaces). +Needs the gh CLI, which GH_TOKEN authenticates. The artifacts come from the +pull request, so they are not trusted: only the numbers in the lines above +are read from them. +""" + +import argparse +import io +import json +import os +import re +import subprocess +import sys +import time +import urllib.parse +import zipfile +from collections import defaultdict +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +# The first line of the comment, which finds it again. +MARKER = "" +# The author of the comments that GITHUB_TOKEN writes. +BOT = "github-actions[bot]" +WORKFLOWS = ".github/workflows" +MEMORY_FILE = "memory.txt" +# memory.txt has two short lines; an artifact much larger than that is not one. +MAX_ARTIFACT_BYTES = 64 * 1024 +MAX_MEMORY_FILE_BYTES = 4 * 1024 + +# The same code does not use the same memory in every CI run. Comparing every +# two of the 107 runs on 2026-10-18 and 19 that shared their source with another +# one (30 groups of 2 to 8 runs of the ten implementations; see docs/memory.md), +# the Home and the Versions PSS never differed by more than 5 MB and 5 % of the +# first number for most of the implementations, and never by more than the +# percentages below for the four others. Only a larger difference counts as a +# change. +MIN_CHANGE_MB = 5 +DEFAULT_TOLERANCE = 0.05 +TOLERANCES = {"compose": 0.08, "flet": 0.08, "gio": 0.12, "flutter": 0.15} + +LINE = re.compile(r"memory \((home|nodejs)\): RSS ([0-9]{1,6}) MB, PSS ([0-9]{1,6}) MB in ([0-9]{1,4}) processes") +IMPLEMENTATION = re.compile(r"[a-z0-9][a-z0-9-]*") +REPOSITORY = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") + + +@dataclass(frozen=True) +class Point: + """The memory at one measuring point, in MB.""" + + rss: int + pss: int + processes: int + + +@dataclass(frozen=True) +class Memory: + """What an implementation uses on the Home page and with the versions of Node.js listed.""" + + home: Point + versions: Point + + +@dataclass +class Row: + """One implementation of the report. + + The state is "measured" (the pull request rebuilt it), "inherited" (it did + not, so the numbers are those of the default branch), "running" or "failed" + (the workflow of the pull request has not ended, or ended without numbers) + or "missing" (no workflow ran and the default branch has no numbers). + """ + + implementation: str + name: str + technology: str = "" + state: str = "missing" + memory: Memory | None = None + baseline: Memory | None = None + run_url: str = "" + baseline_url: str = "" + baseline_sha: str = "" + + @property + def shown(self) -> Memory | None: + """The numbers of the table.""" + if self.state == "measured": + return self.memory + if self.state == "inherited": + return self.baseline + return None + + +class ApiError(RuntimeError): + """A request to the API of GitHub failed.""" + + +def parse_memory(text: str) -> Memory | None: + """Parses memory.txt: one line for the point home and one for nodejs, nothing else.""" + points: dict[str, Point] = {} + for line in text.splitlines(): + line = line.strip() + if not line: + continue + match = LINE.fullmatch(line) + if match is None or match[1] in points: + return None + points[match[1]] = Point(rss=int(match[2]), pss=int(match[3]), processes=int(match[4])) + if points.keys() != {"home", "nodejs"}: + return None + return Memory(home=points["home"], versions=points["nodejs"]) + + +def tolerance(implementation: str, baseline: int) -> float: + """The difference in MB from the baseline that is still the noise of CI.""" + return max(MIN_CHANGE_MB, TOLERANCES.get(implementation, DEFAULT_TOLERANCE) * baseline) + + +def exceeds_noise(implementation: str, baseline: int, current: int) -> bool: + """Whether the difference is a change and not the variation between CI runs.""" + return abs(current - baseline) > tolerance(implementation, baseline) + + +def signed(number: int) -> str: + return f"+{number}" if number > 0 else str(number) + + +def cell(text: str) -> str: + """Text that cannot break out of a table cell.""" + return re.sub(r"\s+", " ", text).replace("|", "\\|").strip() + + +def read_technologies(text: str) -> dict[str, str]: + """The Technology column of the first table of docs/memory.md, by implementation directory.""" + technologies: dict[str, str] = {} + columns: tuple[int, int] | None = None + for line in text.splitlines(): + if not line.startswith("|"): + if columns is not None: + break + continue + cells = [part.strip() for part in line.strip().strip("|").split("|")] + if columns is None: + if "Implementation" in cells and "Technology" in cells: + columns = (cells.index("Implementation"), cells.index("Technology")) + continue + if len(cells) <= max(columns): + continue + link = re.search(r"\]\(\.\./([A-Za-z0-9_-]+)/README\.md", cells[columns[0]]) + if link: + technologies[link[1]] = cells[columns[1]] + return technologies + + +def pss_cell(row: Row, point: str) -> str: + """The PSS of a point, with the difference from the default branch if it was measured.""" + shown = getattr(row.shown, point).pss + if row.state != "measured" or row.baseline is None: + return str(shown) + before = getattr(row.baseline, point).pss + difference = signed(shown - before) + if exceeds_noise(row.implementation, before, shown): + difference = f"**{difference}**" + return f"{shown} ({difference})" + + +def table_cells(row: Row) -> list[str]: + name = row.name + if row.state == "inherited": + name += " (main)" + elif row.state == "measured" and row.baseline is None: + name += " (no baseline)" + shown = row.shown + if shown is None: + label = {"running": "running", "failed": "no result", "missing": "no data"}[row.state] + if row.run_url: + label = f"[{label}]({row.run_url})" + return [cell(name), cell(row.technology), "-", label, "-", "-", "-", "-"] + processes = str(shown.home.processes) + if row.state == "measured" and row.baseline is not None: + # The number of processes of an implementation does not vary between CI runs. + difference = shown.home.processes - row.baseline.home.processes + if difference: + processes += f" (**{signed(difference)}**)" + return [ + cell(name), + cell(row.technology), + processes, + pss_cell(row, "home"), + str(shown.home.rss), + pss_cell(row, "versions"), + str(shown.versions.rss), + signed(shown.versions.pss - shown.home.pss), + ] + + +def sort_key(row: Row) -> tuple[int, int, str]: + group = {"measured": 0, "inherited": 0, "running": 1, "failed": 2, "missing": 3}[row.state] + shown = row.shown + return (group, shown.home.pss if shown else 0, row.name.lower()) + + +def changes_of(row: Row) -> list[str]: + """The differences from the default branch that are more than the noise.""" + if row.state != "measured" or row.memory is None or row.baseline is None: + return [] + changes = [] + for label, point in (("Home PSS", "home"), ("Versions PSS", "versions")): + before = getattr(row.baseline, point).pss + after = getattr(row.memory, point).pss + if exceeds_noise(row.implementation, before, after): + percent = round(100 * (after - before) / before) if before else 0 + changes.append(f"{label} {signed(after - before)} MB ({signed(percent)}%)") + if row.memory.home.processes != row.baseline.home.processes: + changes.append(f"processes {row.baseline.home.processes} -> {row.memory.home.processes}") + return changes + + +def summary(rows: list[Row]) -> list[str]: + """The lines above the table: what changed, and what has no numbers.""" + lines = [] + compared = [row for row in rows if row.state == "measured" and row.baseline is not None] + changed = {row.name: changes_of(row) for row in compared} + changed = {name: parts for name, parts in changed.items() if parts} + if changed: + details = "; ".join(f"{name}: {', '.join(parts)}" for name, parts in changed.items()) + lines.append(f"Changed from main beyond the noise of CI: {details}.") + same = ", ".join(row.name for row in compared if row.name not in changed) + if same and changed: + lines.append(f"Within the noise of CI: {same}.") + elif same: + lines.append(f"No change from main beyond the noise of CI in {same}.") + without_baseline = [row.name for row in rows if row.state == "measured" and row.baseline is None] + for text, names in ( + ("No numbers on main to compare with yet", without_baseline), + ("Still running", [row.name for row in rows if row.state == "running"]), + ("No result, the run failed or was cancelled", [row.name for row in rows if row.state == "failed"]), + ("No numbers anywhere", [row.name for row in rows if row.state == "missing"]), + ): + if names: + lines.append(f"{text}: {', '.join(names)}.") + if all(row.state in ("inherited", "missing") for row in rows): + lines.append("This pull request did not rebuild any implementation, so the numbers are those of main.") + return lines + + +def explanation(rows: list[Row]) -> str: + wider = defaultdict(list) + for row in rows: + if row.implementation in TOLERANCES: + wider[TOLERANCES[row.implementation]].append(row.name) + exceptions = ", ".join( + f"{percent:.0%} for {' and '.join(names)}" for percent, names in sorted(wider.items()) + ) + return ( + "The same code does not use the same memory in every CI run, so a difference counts as a change only " + f"when it is more than {MIN_CHANGE_MB} MB and more than {DEFAULT_TOLERANCE:.0%} of the number on main" + + (f" ({exceptions}, which vary more)" if exceptions else "") + + "." + ) + + +def render(rows: list[Row], *, head_sha: str) -> str: + """The comment.""" + rows = sorted(rows, key=sort_key) + header = [ + "Implementation", + "Technology", + "Processes", + "Home PSS", + "Home RSS", + "Versions PSS", + "Versions RSS", + "The list adds (PSS)", + ] + alignment = ["---", "---", *["---:"] * 6] + body = [table_cells(row) for row in rows] + if not any(row.technology for row in rows): + # Without the table of docs/memory.md there is nothing to put there. + for line in (header, alignment, *body): + del line[1] + table = ["| " + " | ".join(line) + " |" for line in (header, alignment, *body)] + + runs = [f"[{row.name}]({row.run_url})" for row in rows if row.run_url] + baselines = [ + f"[{row.name}]({row.baseline_url}) (`{row.baseline_sha[:7]}`)" for row in rows if row.baseline_url + ] + details = [ + "- Measured by `shared/tools/measure.sh` (Flutter: `flutter/tool/measure.py`) in the workflow of each " + "implementation, on a GitHub-hosted runner with Xvfb and no GPU. These are not the numbers of " + "`docs/memory.md`, which come from another machine: compare the rows of one comment.", + "- " + explanation(rows), + ] + if runs: + details.append("- Runs of this pull request: " + ", ".join(runs) + ".") + if baselines: + details.append("- Latest runs on main: " + ", ".join(baselines) + ".") + + lines = [ + MARKER, + f"### Memory use at `{head_sha[:7]}`", + "", + *[f"- {line}" for line in summary(rows)], + "", + *table, + "", + "In MB, for the application and all of its child processes, sorted by the PSS of the Home page; " + '"Versions" is the SDKs page with the available versions of Node.js listed. A number in parentheses is ' + "the difference from main, in bold when it is more than the noise of CI. " + '"(main)" marks an implementation that this pull request did not rebuild: its numbers are those of main.', + "", + "
", + "Details", + "", + *details, + "
", + "", + ] + return "\n".join(lines) + + +class GitHub: + """The REST API of one repository, through the gh CLI (GH_TOKEN authenticates it).""" + + def __init__(self, repository: str) -> None: + self.repository = repository + + def _request(self, endpoint: str, *, method: str = "GET", payload: Any = None, attempts: int = 3) -> bytes: + command = ["gh", "api", "--method", method, endpoint] + data = None + if payload is not None: + command += ["--input", "-"] + data = json.dumps(payload).encode() + for attempt in range(1, attempts + 1): + result = subprocess.run(command, input=data, capture_output=True, check=False) + if result.returncode == 0: + return result.stdout + error = result.stderr.decode(errors="replace").strip() + transient = re.search(r"HTTP 5[0-9][0-9]|timeout|timed out|connection reset|EOF", error, re.IGNORECASE) + if attempt == attempts or not transient: + raise ApiError(f"{method} {endpoint}: {error}") + time.sleep(2**attempt) + raise AssertionError("unreachable") + + def _endpoint(self, path: str, params: dict[str, Any]) -> str: + endpoint = f"repos/{self.repository}" + (f"/{path}" if path else "") + return endpoint + ("?" + urllib.parse.urlencode(params) if params else "") + + def get(self, path: str, **params: Any) -> Any: + return json.loads(self._request(self._endpoint(path, params))) + + def raw(self, path: str) -> bytes: + """The body of a response that is not JSON, such as the zip of an artifact.""" + return self._request(self._endpoint(path, {})) + + def pages(self, path: str, key: str | None = None, **params: Any) -> Iterator[Any]: + """Every item of a list, 100 at a time.""" + for page in range(1, 11): + data = self.get(path, per_page=100, page=page, **params) + items = data[key] if key else data + yield from items + if len(items) < 100: + return + + def send(self, method: str, path: str, payload: Any = None) -> Any: + # Creating a comment again after a lost response would write it twice. + attempts = 1 if method == "POST" else 3 + body = self._request(self._endpoint(path, {}), method=method, payload=payload, attempts=attempts) + return json.loads(body) if body.strip() else None + + +def log(message: str) -> None: + print(message, file=sys.stderr, flush=True) + + +def read_artifact(gh: GitHub, artifact: dict[str, Any]) -> Memory | None: + """The numbers in an artifact, or None if it is not a valid memory.txt.""" + if artifact.get("expired") or artifact.get("size_in_bytes", 0) > MAX_ARTIFACT_BYTES: + return None + try: + data = gh.raw(f"actions/artifacts/{artifact['id']}/zip") + with zipfile.ZipFile(io.BytesIO(data)) as archive: + files = archive.infolist() + if len(files) != 1 or files[0].filename != MEMORY_FILE: + return None + with archive.open(files[0]) as file: + content = file.read(MAX_MEMORY_FILE_BYTES + 1) + except (zipfile.BadZipFile, ApiError) as error: + log(f"Cannot read the artifact {artifact.get('name')} ({artifact.get('id')}): {error}") + return None + if len(content) > MAX_MEMORY_FILE_BYTES: + return None + return parse_memory(content.decode("utf-8", errors="replace")) + + +def newest(artifacts: list[dict[str, Any]]) -> dict[str, Any] | None: + live = [artifact for artifact in artifacts if not artifact.get("expired")] + return max(live, key=lambda artifact: artifact["created_at"], default=None) + + +def baseline_of(gh: GitHub, implementation: str, default_branch: str) -> tuple[Memory, dict[str, Any]] | None: + """The latest numbers of an implementation that were measured on the default branch.""" + artifacts = gh.get("actions/artifacts", name=f"memory-{implementation}", per_page=100)["artifacts"] + # A pull request from a fork can have a branch called like the default branch. + on_default_branch = [ + artifact + for artifact in artifacts + if (artifact.get("workflow_run") or {}).get("head_branch") == default_branch + and artifact["workflow_run"].get("head_repository_id") == artifact["workflow_run"].get("repository_id") + ] + for artifact in sorted(on_default_branch, key=lambda candidate: candidate["created_at"], reverse=True): + memory = read_artifact(gh, artifact) + if memory is not None: + return memory, artifact + return None + + +def find_pull_requests(gh: GitHub, owner: str | None, branch: str, sha: str) -> list[dict[str, Any]]: + """The open pull requests whose latest commit is sha.""" + pulls: list[dict[str, Any]] = [] + if owner: + pulls = gh.get("pulls", state="open", head=f"{owner}:{branch}", per_page=100) + if not pulls: + pulls = [pull for pull in gh.get(f"commits/{sha}/pulls", per_page=100) if pull["state"] == "open"] + return [pull for pull in pulls if pull["head"]["sha"] == sha] + + +def workflow_path(implementation: str) -> str: + return f"{WORKFLOWS}/{implementation}.yml" + + +def active_workflows(gh: GitHub, implementations: list[str]) -> dict[str, dict[str, Any]]: + """The workflows of the implementations, by implementation; one that is gone or disabled is left out.""" + workflows = {workflow["path"]: workflow for workflow in gh.get("actions/workflows", per_page=100)["workflows"]} + return { + implementation: workflows[workflow_path(implementation)] + for implementation in implementations + if workflows.get(workflow_path(implementation), {}).get("state") == "active" + } + + +def latest_runs(gh: GitHub, head_sha: str) -> dict[str, dict[str, Any]]: + """The latest run of every workflow of the pull request for a commit, by the path of the workflow.""" + runs: dict[str, dict[str, Any]] = {} + for run in gh.get("actions/runs", head_sha=head_sha, event="pull_request", per_page=100)["workflow_runs"]: + if run["path"] not in runs or run["id"] > runs[run["path"]]["id"]: + runs[run["path"]] = run + return runs + + +def unfinished( + workflows: dict[str, dict[str, Any]], runs: dict[str, dict[str, Any]], finished_run: int | None +) -> list[str]: + """The names of the workflows with a run that has not ended, other than the run that has just ended.""" + return [ + workflow["name"] + for implementation, workflow in workflows.items() + if (run := runs.get(workflow_path(implementation))) + and run["status"] != "completed" + and run["id"] != finished_run + ] + + +def collect( + gh: GitHub, + *, + workflows: dict[str, dict[str, Any]], + runs: dict[str, dict[str, Any]], + technologies: dict[str, str], + default_branch: str, + finished_run: int | None, +) -> list[Row]: + """The rows of the report: the numbers of the pull request and of the default branch.""" + rows = [] + for implementation, workflow in workflows.items(): + row = Row(implementation, workflow["name"], technologies.get(implementation, "")) + run = runs.get(workflow_path(implementation)) + if run is None: + row.state = "inherited" + else: + row.run_url = run["html_url"] + if run["status"] != "completed" and run["id"] != finished_run: + row.state = "running" + else: + artifacts = gh.get( + f"actions/runs/{run['id']}/artifacts", name=f"memory-{implementation}", per_page=100 + )["artifacts"] + artifact = newest(artifacts) + row.memory = read_artifact(gh, artifact) if artifact else None + row.state = "measured" if row.memory else "failed" + if row.state in ("measured", "inherited"): + baseline = baseline_of(gh, implementation, default_branch) + if baseline is not None: + memory, artifact = baseline + row.baseline = memory + row.baseline_sha = artifact["workflow_run"]["head_sha"] + row.baseline_url = f"https://github.com/{gh.repository}/actions/runs/{artifact['workflow_run']['id']}" + elif row.state == "inherited": + row.state = "missing" + rows.append(row) + return rows + + +def same_text(left: str, right: str) -> bool: + return left.replace("\r\n", "\n").strip() == right.replace("\r\n", "\n").strip() + + +def upsert_comment(gh: GitHub, number: int, body: str) -> str: + """Writes the comment of the pull request, or replaces the one that an earlier run wrote.""" + mine = sorted( + ( + comment + for comment in gh.pages(f"issues/{number}/comments") + if (comment.get("user") or {}).get("login") == BOT and comment["body"].startswith(MARKER) + ), + key=lambda comment: comment["id"], + ) + if not mine: + gh.send("POST", f"issues/{number}/comments", {"body": body}) + return "created" + result = "unchanged" + if not same_text(mine[0]["body"], body): + gh.send("PATCH", f"issues/comments/{mine[0]['id']}", {"body": body}) + result = "updated" + # Two runs at the same time can both have created one. + for extra in mine[1:]: + gh.send("DELETE", f"issues/comments/{extra['id']}") + return result + + +def report( + gh: GitHub, + *, + implementations: list[str], + technologies: dict[str, str], + run_id: int | None, + pull_request: int | None, + dry_run: bool, +) -> int: + # Most runs end here, on the way to the one that finds that all of them have: keep it to a few requests. + if run_id is not None: + run = gh.get(f"actions/runs/{run_id}") + if run["event"] != "pull_request": + log(f"The run {run_id} is a {run['event']} run, not one of a pull request.") + return 0 + head_sha = run["head_sha"] + owner = ((run.get("head_repository") or {}).get("owner") or {}).get("login") + pulls = find_pull_requests(gh, owner, run["head_branch"], head_sha) + else: + pull = gh.get(f"pulls/{pull_request}") + if pull["state"] != "open": + log(f"The pull request {pull_request} is not open.") + return 0 + head_sha = pull["head"]["sha"] + pulls = [pull] + if not pulls: + log(f"No open pull request has {head_sha[:7]} as its latest commit; there is nothing to report on.") + return 0 + workflows = active_workflows(gh, implementations) + if not workflows: + log("None of the implementations has a workflow.") + return 0 + runs = latest_runs(gh, head_sha) + waiting = unfinished(workflows, runs, run_id) if run_id is not None else [] + if waiting: + log(f"Waiting for {', '.join(waiting)}; the report is written when they have ended.") + return 0 + + rows = collect( + gh, + workflows=workflows, + runs=runs, + technologies=technologies, + default_branch=gh.get("")["default_branch"], + finished_run=run_id, + ) + body = render(rows, head_sha=head_sha) + summary_file = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_file: + with open(summary_file, "a", encoding="utf-8") as file: + file.write(body.removeprefix(MARKER + "\n")) + if dry_run: + print(body) + return 0 + for pull in pulls: + # A commit that came while the numbers were collected has its own report. + current = gh.get(f"pulls/{pull['number']}") + if current["state"] != "open" or current["head"]["sha"] != head_sha: + log(f"The pull request {pull['number']} has moved on from {head_sha[:7]}.") + continue + log(f"Pull request {pull['number']}: comment {upsert_comment(gh, pull['number'], body)}.") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repository", default=os.environ.get("GITHUB_REPOSITORY", ""), help="OWNER/NAME") + parser.add_argument( + "--implementations", + default=os.environ.get("MEMORY_IMPLEMENTATIONS", ""), + help="the implementations, such as 'mygo tauri', separated by spaces", + ) + parser.add_argument("--docs", type=Path, default=Path("docs/memory.md"), help="the page with the Technology column") + parser.add_argument("--dry-run", action="store_true", help="print the comment instead of writing it") + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument("--run", type=int, help="the ID of the workflow run that has just ended") + target.add_argument("--pull-request", type=int, help="the number of the pull request") + arguments = parser.parse_args(argv) + + implementations = arguments.implementations.split() + if not REPOSITORY.fullmatch(arguments.repository): + parser.error("--repository (or GITHUB_REPOSITORY) must be OWNER/NAME") + if not implementations or not all(IMPLEMENTATION.fullmatch(name) for name in implementations): + parser.error("--implementations (or MEMORY_IMPLEMENTATIONS) must list names such as 'mygo tauri'") + technologies = {} + if arguments.docs.is_file(): + technologies = read_technologies(arguments.docs.read_text(encoding="utf-8")) + try: + return report( + GitHub(arguments.repository), + implementations=implementations, + technologies=technologies, + run_id=arguments.run, + pull_request=arguments.pull_request, + dry_run=arguments.dry_run, + ) + except ApiError as error: + if "Resource not accessible by integration" in str(error): + # A token that cannot write, such as that of a run for a bot's pull request: the report is + # already in the summary of the run, which is as far as it can go. + message = f"The token of this run cannot write the comment; the report is in the summary: {error}" + # The encoding of the data of a workflow command. + message = message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + print(f"::warning::{message}") + return 0 + log(f"memory_report.py: {error}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test_memory_report.py b/.github/scripts/test_memory_report.py new file mode 100644 index 0000000..51b8667 --- /dev/null +++ b/.github/scripts/test_memory_report.py @@ -0,0 +1,878 @@ +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests of memory_report.py: python3 -m unittest discover -s .github/scripts""" + +import contextlib +import io +import os +import re +import subprocess +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest import mock + +import memory_report as report +from memory_report import Memory, Point, Row + +REPOSITORY = "version-fox/vfox-plus" +HEAD = "a" * 40 +MAIN = "b" * 40 + + +def setUpModule(): + # CI sets these. The tests must not write to the summary of the job that runs them, or + # take the repository and the implementations from it. + patcher = mock.patch.dict(os.environ) + patcher.start() + unittest.addModuleCleanup(patcher.stop) + for name in ("GITHUB_STEP_SUMMARY", "GITHUB_REPOSITORY", "MEMORY_IMPLEMENTATIONS", "GH_TOKEN"): + os.environ.pop(name, None) + + +def memory_text(home=(55, 42, 1), versions=(64, 49, 1)) -> str: + return ( + f"memory (home): RSS {home[0]} MB, PSS {home[1]} MB in {home[2]} processes\n" + f"memory (nodejs): RSS {versions[0]} MB, PSS {versions[1]} MB in {versions[2]} processes\n" + ) + + +def make_zip(files: dict[str, str | bytes]) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in files.items(): + archive.writestr(name, content) + return buffer.getvalue() + + +def memory(home_pss=100, versions_pss=120, processes=1) -> Memory: + return Memory(Point(home_pss + 10, home_pss, processes), Point(versions_pss + 10, versions_pss, processes)) + + +class FakeGitHub: + """The parts of the API of GitHub that the report reads and writes.""" + + repository = REPOSITORY + + def __init__(self) -> None: + self.default_branch = "main" + self.workflows: list[dict] = [] + self.runs: list[dict] = [] + self.artifacts: list[dict] = [] + self.zips: dict[int, bytes] = {} + self.pulls: dict[int, dict] = {} + self.comments: dict[int, list[dict]] = {} + self.writes: list[tuple[str, str]] = [] + self._ids = 1000 + + def next_id(self) -> int: + self._ids += 1 + return self._ids + + # What the tests set up. + def add_workflow(self, implementation: str, name: str | None = None, state: str = "active") -> None: + self.workflows.append( + { + "path": f".github/workflows/{implementation}.yml", + "name": name or implementation.capitalize(), + "state": state, + } + ) + + def add_pull_request(self, number: int = 7, sha: str = HEAD, state: str = "open") -> None: + head = {"sha": sha, "ref": "feature", "owner": "contributor"} + self.pulls[number] = {"number": number, "state": state, "head": head} + + def add_run( + self, + implementation: str, + text: str | bytes | None = None, + *, + status: str = "completed", + conclusion: str | None = "success", + sha: str = HEAD, + event: str = "pull_request", + run_id: int | None = None, + artifact_files: dict | None = None, + ) -> int: + run_id = run_id or self.next_id() + self.runs.append( + { + "id": run_id, + "path": f".github/workflows/{implementation}.yml", + "name": implementation.capitalize(), + "event": event, + "status": status, + "conclusion": conclusion, + "head_sha": sha, + "head_branch": "feature", + "html_url": f"https://github.com/{REPOSITORY}/actions/runs/{run_id}", + "head_repository": {"owner": {"login": "contributor"}}, + } + ) + if text is not None or artifact_files is not None: + files = artifact_files if artifact_files is not None else {"memory.txt": text} + self.add_artifact(implementation, files, run_id=run_id, branch="feature", sha=sha) + return run_id + + def add_artifact( + self, + implementation: str, + files: dict | str, + *, + run_id: int | None = None, + branch: str = "main", + sha: str = MAIN, + created: str = "2026-10-20T10:00:00Z", + expired: bool = False, + head_repository_id: int = 1, + size: int | None = None, + ) -> int: + files = {"memory.txt": files} if isinstance(files, str) else files + archive = make_zip(files) + artifact_id = self.next_id() + self.zips[artifact_id] = archive + self.artifacts.append( + { + "id": artifact_id, + "name": f"memory-{implementation}", + "size_in_bytes": len(archive) if size is None else size, + "expired": expired, + "created_at": created, + "workflow_run": { + "id": run_id or self.next_id(), + "head_branch": branch, + "head_sha": sha, + "head_repository_id": head_repository_id, + "repository_id": 1, + }, + } + ) + return artifact_id + + # The API. + def get(self, path: str, **params): + if path == "": + return {"default_branch": self.default_branch} + if path == "actions/workflows": + return {"workflows": self.workflows} + if path == "actions/runs": + return { + "workflow_runs": [ + run + for run in self.runs + if run["head_sha"] == params["head_sha"] and run["event"] == params["event"] + ] + } + if match := re.fullmatch(r"actions/runs/(\d+)", path): + return next(run for run in self.runs if run["id"] == int(match[1])) + if match := re.fullmatch(r"actions/runs/(\d+)/artifacts", path): + return { + "artifacts": [ + artifact + for artifact in self.artifacts + if artifact["workflow_run"]["id"] == int(match[1]) and artifact["name"] == params["name"] + ] + } + if path == "actions/artifacts": + return {"artifacts": [a for a in self.artifacts if a["name"] == params["name"]]} + if path == "pulls": + owner, _, branch = params["head"].partition(":") + return [ + pull + for pull in self.pulls.values() + if pull["state"] == params["state"] and pull["head"]["owner"] == owner and pull["head"]["ref"] == branch + ] + if match := re.fullmatch(r"pulls/(\d+)", path): + return self.pulls[int(match[1])] + if match := re.fullmatch(r"commits/(\w+)/pulls", path): + return [pull for pull in self.pulls.values() if pull["head"]["sha"] == match[1]] + raise AssertionError(f"Unexpected GET {path} {params}") + + def pages(self, path: str, key: str | None = None, **params): + match = re.fullmatch(r"issues/(\d+)/comments", path) + assert match, path + yield from list(self.comments.get(int(match[1]), [])) + + def raw(self, path: str) -> bytes: + match = re.fullmatch(r"actions/artifacts/(\d+)/zip", path) + assert match, path + return self.zips[int(match[1])] + + def send(self, method: str, path: str, payload=None): + self.writes.append((method, path)) + if match := re.fullmatch(r"issues/(\d+)/comments", path): + comment = {"id": self.next_id(), "body": payload["body"], "user": {"login": report.BOT}} + self.comments.setdefault(int(match[1]), []).append(comment) + return comment + if match := re.fullmatch(r"issues/comments/(\d+)", path): + for comments in self.comments.values(): + for comment in list(comments): + if comment["id"] == int(match[1]): + if method == "PATCH": + comment["body"] = payload["body"] + else: + comments.remove(comment) + return None + raise AssertionError(f"Unexpected {method} {path}") + + +class ParseMemoryTest(unittest.TestCase): + def test_reads_both_points(self): + self.assertEqual( + report.parse_memory(memory_text((55, 42, 1), (64, 49, 3))), + Memory(Point(55, 42, 1), Point(64, 49, 3)), + ) + + def test_the_order_and_blank_lines_do_not_matter(self): + text = "\n" + memory_text().splitlines()[1] + "\n\n" + memory_text().splitlines()[0] + "\n" + self.assertIsNotNone(report.parse_memory(text)) + + def test_needs_both_points(self): + self.assertIsNone(report.parse_memory(memory_text().splitlines()[0])) + self.assertIsNone(report.parse_memory("")) + + def test_rejects_a_point_twice(self): + line = memory_text().splitlines()[0] + self.assertIsNone(report.parse_memory(f"{line}\n{line}\n{memory_text().splitlines()[1]}")) + + def test_rejects_anything_else(self): + self.assertIsNone(report.parse_memory(memory_text() + "e2e: MEASURE home\n")) + self.assertIsNone(report.parse_memory(memory_text().replace("MB, PSS", "MB, PSS -"))) + self.assertIsNone(report.parse_memory(memory_text().replace("home", "other"))) + self.assertIsNone(report.parse_memory(memory_text((10**6, 1, 1), (1, 1, 1)))) + + def test_rejects_digits_that_are_not_ascii(self): + self.assertIsNone(report.parse_memory(memory_text().replace("55", "٥٥"))) + + +class ToleranceTest(unittest.TestCase): + def test_a_small_implementation_is_judged_by_the_absolute_floor(self): + self.assertFalse(report.exceeds_noise("mygo", 42, 47)) + self.assertTrue(report.exceeds_noise("mygo", 42, 48)) + self.assertTrue(report.exceeds_noise("mygo", 42, 36)) + self.assertFalse(report.exceeds_noise("mygo", 42, 37)) + + def test_a_large_implementation_is_judged_by_the_percentage(self): + self.assertFalse(report.exceeds_noise("tauri", 300, 315)) + self.assertTrue(report.exceeds_noise("tauri", 300, 316)) + + def test_the_noisy_implementations_get_more_room(self): + self.assertFalse(report.exceeds_noise("flutter", 350, 350 + 52)) + self.assertTrue(report.exceeds_noise("flutter", 350, 350 + 53)) + self.assertFalse(report.exceeds_noise("gio", 200, 224)) + self.assertTrue(report.exceeds_noise("gio", 200, 225)) + + def test_no_difference_is_no_change(self): + for implementation in ("mygo", "gio", "unknown"): + self.assertFalse(report.exceeds_noise(implementation, 100, 100)) + + +class TechnologiesTest(unittest.TestCase): + PAGE = """# Memory use + +Text with a | pipe. + +| Implementation | Technology | Processes | +| -------------------------------- | ------------------ | --------: | +| [mygo](../mygo/README.md#memory) | mygo 0.5, Go, GTK | 1 | +| [Uno](../uno/README.md#memory) | Uno Platform 6 | 1 | + +| Implementation | Other | +| --- | --- | +| [later](../later/README.md) | not read | +""" + + def test_reads_the_first_table_by_directory(self): + self.assertEqual( + report.read_technologies(self.PAGE), {"mygo": "mygo 0.5, Go, GTK", "uno": "Uno Platform 6"} + ) + + def test_a_page_without_the_table_gives_nothing(self): + self.assertEqual(report.read_technologies("# Nothing\n\n| a | b |\n| - | - |\n"), {}) + + def test_reads_the_page_of_this_repository(self): + page = Path(__file__).resolve().parents[2] / "docs" / "memory.md" + technologies = report.read_technologies(page.read_text(encoding="utf-8")) + self.assertTrue(technologies) + self.assertTrue(all(technologies.values())) + + +class RenderTest(unittest.TestCase): + def rows(self) -> list[Row]: + return [ + Row("tauri", "Tauri", "Tauri 2, Rust", "measured", memory(283, 305, 3), memory(283, 305, 3)), + Row("mygo", "mygo", "mygo 0.5, Go", "inherited", None, memory(42, 49, 1)), + Row("uno", "Uno", "", "measured", memory(101, 128), memory(101, 127)), + ] + + def render(self, rows: list[Row]) -> str: + return report.render(rows, head_sha=HEAD) + + def table(self, text: str) -> list[list[str]]: + lines = [line for line in text.splitlines() if line.startswith("|")] + return [[cell.strip() for cell in line.strip("|").split("|")] for line in lines] + + def test_starts_with_the_marker_that_finds_it_again(self): + text = self.render(self.rows()) + self.assertEqual(text.splitlines()[0], report.MARKER) + self.assertIn("`aaaaaaa`", text.splitlines()[1]) + + def test_is_sorted_by_the_pss_of_the_home_page(self): + table = self.table(self.render(self.rows())) + self.assertEqual([line[0] for line in table[2:]], ["mygo (main)", "Uno", "Tauri"]) + + def test_shows_the_difference_from_main_for_what_was_measured_only(self): + table = self.table(self.render(self.rows())) + by_name = {line[0]: line for line in table[2:]} + self.assertEqual(by_name["Tauri"][3], "283 (0)") + self.assertEqual(by_name["Uno"][5], "128 (+1)") + self.assertEqual(by_name["mygo (main)"][3], "42") + + def test_the_list_adds_what_the_versions_use_beyond_the_home_page(self): + table = self.table(self.render(self.rows())) + self.assertEqual({line[0]: line[-1] for line in table[2:]}, {"mygo (main)": "+7", "Uno": "+27", "Tauri": "+22"}) + + def test_a_change_beyond_the_noise_is_bold_and_in_the_summary(self): + rows = [Row("tauri", "Tauri", "", "measured", memory(330, 305), memory(283, 305))] + text = self.render(rows) + self.assertIn("330 (**+47**)", text) + self.assertIn("- Changed from main beyond the noise of CI: Tauri: Home PSS +47 MB (+17%).", text) + self.assertNotIn("Within the noise", text) + + def test_a_decrease_counts_too(self): + rows = [Row("tauri", "Tauri", "", "measured", memory(200, 305), memory(283, 305))] + self.assertIn("200 (**-83**)", self.render(rows)) + + def test_a_change_in_the_number_of_processes_is_a_change(self): + rows = [Row("tauri", "Tauri", "", "measured", memory(283, 305, 4), memory(283, 305, 3))] + text = self.render(rows) + self.assertIn("4 (**+1**)", text) + self.assertIn("Tauri: processes 3 -> 4", text) + + def test_unchanged_implementations_are_listed_apart_from_the_changed_one(self): + rows = [ + Row("tauri", "Tauri", "", "measured", memory(330, 305), memory(283, 305)), + Row("uno", "Uno", "", "measured", memory(101, 127), memory(101, 127)), + ] + text = self.render(rows) + self.assertIn("- Within the noise of CI: Uno.", text) + + def test_says_so_when_nothing_changed(self): + rows = [Row("uno", "Uno", "", "measured", memory(101, 127), memory(101, 127))] + self.assertIn("- No change from main beyond the noise of CI in Uno.", self.render(rows)) + + def test_an_implementation_without_numbers_on_main_has_nothing_to_compare_with(self): + rows = [Row("uno", "Uno", "", "measured", memory(101, 127), None)] + text = self.render(rows) + self.assertIn("Uno (no baseline)", text) + self.assertIn("- No numbers on main to compare with yet: Uno.", text) + self.assertIn("| 101 |", text) + + def test_failed_and_running_implementations_come_last_with_a_link(self): + rows = self.rows() + [ + Row("flet", "Flet", "", "failed", run_url="https://example.test/failed"), + Row("gio", "Gio", "", "running", run_url="https://example.test/running"), + Row("wails", "Wails", "", "missing"), + ] + text = self.render(rows) + names = [line[0] for line in self.table(text)[2:]] + self.assertEqual(names, ["mygo (main)", "Uno", "Tauri", "Gio", "Flet", "Wails"]) + self.assertIn("[no result](https://example.test/failed)", text) + self.assertIn("[running](https://example.test/running)", text) + self.assertIn("- Still running: Gio.", text) + self.assertIn("- No result, the run failed or was cancelled: Flet.", text) + self.assertIn("- No numbers anywhere: Wails.", text) + + def test_says_when_the_pull_request_rebuilt_nothing(self): + rows = [Row("mygo", "mygo", "", "inherited", None, memory(42, 49))] + self.assertIn("did not rebuild any implementation", self.render(rows)) + self.assertNotIn("did not rebuild any implementation", self.render(self.rows())) + + def test_drops_the_technology_column_without_technologies(self): + rows = [Row("uno", "Uno", "", "measured", memory(101, 127), memory(101, 127))] + table = self.table(self.render(rows)) + self.assertEqual(table[0][:3], ["Implementation", "Processes", "Home PSS"]) + self.assertEqual({len(line) for line in table}, {7}) + + def test_keeps_the_technology_column_with_technologies(self): + table = self.table(self.render(self.rows())) + self.assertEqual(table[0][:2], ["Implementation", "Technology"]) + self.assertEqual({len(line) for line in table}, {8}) + + def test_a_cell_cannot_break_the_table(self): + rows = [Row("uno", "Un|o\n", "a | b\nc", "measured", memory(101, 127), memory(101, 127))] + table = self.table(self.render(rows).replace("\\|", "/")) + self.assertEqual({len(line) for line in table}, {8}) + + def test_links_the_runs(self): + rows = [ + Row( + "uno", + "Uno", + "", + "measured", + memory(101, 127), + memory(101, 127), + run_url="https://example.test/pr", + baseline_url="https://example.test/main", + baseline_sha=MAIN, + ) + ] + text = self.render(rows) + self.assertIn("[Uno](https://example.test/pr)", text) + self.assertIn("[Uno](https://example.test/main) (`bbbbbbb`)", text) + + def test_explains_the_noise_that_it_allows(self): + rows = [ + Row("flutter", "Flutter", "", "measured", memory(300, 310), memory(300, 310)), + Row("gio", "Gio", "", "measured", memory(200, 210), memory(200, 210)), + ] + text = self.render(rows) + self.assertIn("more than 5 MB and more than 5% of the number on main", text) + self.assertIn("12% for Gio, 15% for Flutter, which vary more", text) + + def test_is_ascii(self): + self.render(self.rows()).encode("ascii") + + +class ReadArtifactTest(unittest.TestCase): + def read(self, files, **overrides): + github = FakeGitHub() + artifact_id = github.add_artifact("uno", files, **overrides) + artifact = github.artifacts[-1] + self.assertEqual(artifact["id"], artifact_id) + return report.read_artifact(github, artifact) + + def test_reads_memory_txt(self): + self.assertEqual(self.read(memory_text((55, 42, 1), (64, 49, 1))), Memory(Point(55, 42, 1), Point(64, 49, 1))) + + def test_rejects_other_files(self): + self.assertIsNone(self.read({"other.txt": memory_text()})) + self.assertIsNone(self.read({"memory.txt": memory_text(), "extra.txt": "x"})) + self.assertIsNone(self.read({"../memory.txt": memory_text()})) + + def test_rejects_an_expired_or_a_large_artifact_without_downloading_it(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text(), expired=True) + github.add_artifact("uno", memory_text(), size=report.MAX_ARTIFACT_BYTES + 1) + github.raw = mock.Mock(side_effect=AssertionError("downloaded")) + for artifact in github.artifacts: + self.assertIsNone(report.read_artifact(github, artifact)) + + def test_rejects_a_large_file_inside_the_zip(self): + self.assertIsNone(self.read(memory_text() + "x" * (report.MAX_MEMORY_FILE_BYTES + 1))) + + def test_rejects_what_is_not_a_zip(self): + github = FakeGitHub() + artifact_id = github.add_artifact("uno", memory_text()) + github.zips[artifact_id] = b"not a zip" + with contextlib.redirect_stderr(io.StringIO()): + self.assertIsNone(report.read_artifact(github, github.artifacts[-1])) + + def test_rejects_wrong_numbers(self): + self.assertIsNone(self.read("memory (home): RSS many MB, PSS 1 MB in 1 processes\n")) + + +class BaselineTest(unittest.TestCase): + def test_takes_the_newest_valid_artifact_of_the_default_branch(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text((10, 9, 1), (10, 9, 1)), created="2026-10-18T00:00:00Z", sha="1" * 40) + github.add_artifact("uno", memory_text((20, 19, 1), (20, 19, 1)), created="2026-10-20T00:00:00Z", sha="2" * 40) + baseline, artifact = report.baseline_of(github, "uno", "main") + self.assertEqual(baseline.home.pss, 19) + self.assertEqual(artifact["workflow_run"]["head_sha"], "2" * 40) + + def test_ignores_pull_requests_forks_and_expired_artifacts(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text(), branch="feature", created="2026-10-22T00:00:00Z") + github.add_artifact("uno", memory_text(), head_repository_id=2, created="2026-10-22T00:00:00Z") + github.add_artifact("uno", memory_text(), expired=True, created="2026-10-22T00:00:00Z") + self.assertIsNone(report.baseline_of(github, "uno", "main")) + + def test_falls_back_to_an_older_artifact_when_the_newest_is_invalid(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text((10, 9, 1), (10, 9, 1)), created="2026-10-18T00:00:00Z") + github.add_artifact("uno", "garbage", created="2026-10-20T00:00:00Z") + self.assertEqual(report.baseline_of(github, "uno", "main")[0].home.pss, 9) + + def test_only_reads_the_artifacts_of_the_implementation(self): + github = FakeGitHub() + github.add_artifact("tauri", memory_text()) + self.assertIsNone(report.baseline_of(github, "uno", "main")) + + def test_ignores_an_artifact_that_no_workflow_run_made(self): + github = FakeGitHub() + github.add_artifact("uno", memory_text()) + github.artifacts[-1]["workflow_run"] = None + self.assertIsNone(report.baseline_of(github, "uno", "main")) + + +class PullRequestTest(unittest.TestCase): + def test_finds_it_by_the_branch_of_the_run(self): + github = FakeGitHub() + github.add_pull_request(7) + pulls = report.find_pull_requests(github, "contributor", "feature", HEAD) + self.assertEqual([pull["number"] for pull in pulls], [7]) + + def test_finds_it_by_the_commit_without_the_owner(self): + github = FakeGitHub() + github.add_pull_request(7) + self.assertEqual([pull["number"] for pull in report.find_pull_requests(github, None, "feature", HEAD)], [7]) + + def test_a_pull_request_that_has_a_newer_commit_is_not_the_one_of_the_run(self): + github = FakeGitHub() + github.add_pull_request(7, sha="c" * 40) + self.assertEqual(report.find_pull_requests(github, "contributor", "feature", HEAD), []) + + def test_a_closed_pull_request_is_not_found(self): + github = FakeGitHub() + github.add_pull_request(7, state="closed") + self.assertEqual(report.find_pull_requests(github, "contributor", "feature", HEAD), []) + + +class UpsertCommentTest(unittest.TestCase): + def comment(self, github, body, login=report.BOT, number=7): + comment = {"id": github.next_id(), "body": body, "user": {"login": login}} + github.comments.setdefault(number, []).append(comment) + return comment + + def test_creates_the_first_comment(self): + github = FakeGitHub() + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nnew"), "created") + self.assertEqual(github.writes, [("POST", "issues/7/comments")]) + + def test_replaces_the_earlier_comment_instead_of_adding_another(self): + github = FakeGitHub() + old = self.comment(github, f"{report.MARKER}\nold") + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nnew"), "updated") + self.assertEqual(github.writes, [("PATCH", f"issues/comments/{old['id']}")]) + self.assertEqual([c["body"] for c in github.comments[7]], [f"{report.MARKER}\nnew"]) + + def test_leaves_an_identical_comment_alone(self): + github = FakeGitHub() + self.comment(github, f"{report.MARKER}\r\nsame\r\n") + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nsame\n"), "unchanged") + self.assertEqual(github.writes, []) + + def test_keeps_the_oldest_comment_and_deletes_the_rest(self): + github = FakeGitHub() + first = self.comment(github, f"{report.MARKER}\nfirst") + second = self.comment(github, f"{report.MARKER}\nsecond") + report.upsert_comment(github, 7, f"{report.MARKER}\nnew") + self.assertEqual( + github.writes, [("PATCH", f"issues/comments/{first['id']}"), ("DELETE", f"issues/comments/{second['id']}")] + ) + self.assertEqual([c["body"] for c in github.comments[7]], [f"{report.MARKER}\nnew"]) + + def test_never_touches_the_comments_of_other_people(self): + github = FakeGitHub() + theirs = self.comment(github, f"{report.MARKER}\nquoted by a person", login="contributor") + other = self.comment(github, "a remark by the bot") + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nnew"), "created") + self.assertEqual(github.writes, [("POST", "issues/7/comments")]) + self.assertIn(theirs, github.comments[7]) + self.assertIn(other, github.comments[7]) + + def test_tolerates_a_comment_of_a_deleted_user(self): + github = FakeGitHub() + github.comments[7] = [{"id": 1, "body": "text", "user": None}] + self.assertEqual(report.upsert_comment(github, 7, f"{report.MARKER}\nnew"), "created") + + +class ReportTest(unittest.TestCase): + """The whole report, against a fake GitHub.""" + + # The numbers on main: (name of the workflow, Home as RSS, PSS, processes, Versions as RSS, PSS, processes). + MAIN_NUMBERS = { + "mygo": ("mygo", (55, 42, 1), (64, 49, 1)), + "tauri": ("Tauri", (470, 283, 3), (492, 305, 3)), + "uno": ("Uno", (110, 101, 1), (136, 127, 1)), + } + + def setUp(self): + self.github = FakeGitHub() + for implementation, (name, home, versions) in self.MAIN_NUMBERS.items(): + self.github.add_workflow(implementation, name) + self.github.add_artifact(implementation, memory_text(home, versions)) + self.github.add_pull_request(7) + + def run_report(self, **overrides): + arguments = dict( + implementations=["mygo", "tauri", "uno"], + technologies={"mygo": "mygo 0.5, Go"}, + run_id=None, + pull_request=7, + dry_run=False, + ) + arguments.update(overrides) + output, errors = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + status = report.report(self.github, **arguments) + return status, output.getvalue(), errors.getvalue() + + def body(self) -> str: + (comment,) = self.github.comments[7] + return comment["body"] + + def test_comments_with_the_measured_implementation_and_main_for_the_rest(self): + run = self.github.add_run("mygo", memory_text((56, 43, 1), (65, 50, 1))) + status, _, _ = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + table = [line for line in self.body().splitlines() if line.startswith("|")] + self.assertEqual([line.split("|")[1].strip() for line in table[2:]], ["mygo", "Uno (main)", "Tauri (main)"]) + self.assertIn("43 (+1)", self.body()) + self.assertIn("No change from main beyond the noise of CI in mygo.", self.body()) + + def test_waits_for_the_other_implementations_that_are_still_running(self): + run = self.github.add_run("mygo", memory_text()) + self.github.add_run("tauri", status="in_progress", conclusion=None) + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + self.assertIn("Waiting for Tauri", errors) + self.assertEqual(self.github.comments, {}) + + def test_waiting_takes_few_requests(self): + # Every workflow that ends starts a report, and all but the last one only wait. + run = self.github.add_run("mygo", memory_text()) + self.github.add_run("tauri", status="in_progress", conclusion=None) + paths = [] + original = self.github.get + + def get(path, **params): + paths.append(path) + return original(path, **params) + + self.github.get = get + self.github.raw = mock.Mock(side_effect=AssertionError("downloaded an artifact")) + self.run_report(run_id=run, pull_request=None) + self.assertLessEqual(len(paths), 6, paths) + self.assertFalse([path for path in paths if "artifacts" in path], paths) + + def test_the_last_run_to_end_writes_the_report_with_all_of_them(self): + self.github.add_run("tauri", memory_text((470, 283, 3), (492, 305, 3))) + run = self.github.add_run("mygo", memory_text()) + self.run_report(run_id=run, pull_request=None) + table = [line for line in self.body().splitlines() if line.startswith("|")] + self.assertEqual([line.split("|")[1].strip() for line in table[2:]], ["mygo", "Uno (main)", "Tauri"]) + + def test_the_run_that_ended_counts_as_ended_even_if_the_list_is_late(self): + run = self.github.add_run("mygo", memory_text(), status="in_progress", conclusion=None) + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertNotIn("Waiting", errors) + self.assertIn("mygo", self.body()) + + def test_a_failed_run_is_reported_as_such(self): + self.github.add_run("tauri", conclusion="failure") + run = self.github.add_run("mygo", memory_text()) + self.run_report(run_id=run, pull_request=None) + self.assertIn("No result, the run failed or was cancelled: Tauri.", self.body()) + + def test_an_invalid_artifact_is_reported_as_no_result(self): + run = self.github.add_run("mygo", "not the numbers") + self.run_report(run_id=run, pull_request=None) + self.assertIn("No result, the run failed or was cancelled: mygo.", self.body()) + + def test_the_second_report_replaces_the_comment_of_the_first(self): + run = self.github.add_run("mygo", memory_text((56, 43, 1), (65, 50, 1))) + self.run_report(run_id=run, pull_request=None) + self.github.pulls[7]["head"]["sha"] = "c" * 40 + later = self.github.add_run("mygo", memory_text((60, 47, 1), (70, 54, 1)), sha="c" * 40) + self.run_report(run_id=later, pull_request=None) + self.assertEqual(len(self.github.comments[7]), 1) + self.assertIn("47 (+5)", self.body()) + self.assertIn("`ccccccc`", self.body()) + self.assertEqual([method for method, _ in self.github.writes], ["POST", "PATCH"]) + + def test_ignores_the_run_of_a_commit_that_is_no_longer_the_latest(self): + run = self.github.add_run("mygo", memory_text()) + self.github.pulls[7]["head"]["sha"] = "c" * 40 + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + self.assertIn("No open pull request has aaaaaaa", errors) + self.assertEqual(self.github.writes, []) + + def test_does_not_comment_when_the_commit_changes_during_the_report(self): + run = self.github.add_run("mygo", memory_text()) + original = self.github.get + + def get(path, **params): + # The first read of the pull request is the one that finds it; later reads see a push. + if path == "pulls/7": + self.github.pulls[7]["head"]["sha"] = "c" * 40 + return original(path, **params) + + self.github.get = get + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + self.assertIn("has moved on", errors) + self.assertEqual(self.github.writes, []) + + def test_ignores_runs_that_are_not_of_a_pull_request(self): + run = self.github.add_run("mygo", memory_text(), event="push", sha=MAIN) + status, _, errors = self.run_report(run_id=run, pull_request=None) + self.assertEqual(status, 0) + self.assertIn("not one of a pull request", errors) + self.assertEqual(self.github.writes, []) + + def test_skips_a_workflow_that_no_longer_exists(self): + self.github.workflows[1]["state"] = "deleted" + self.run_report() + self.assertNotIn("Tauri", self.body()) + + def test_an_implementation_without_a_workflow_is_skipped(self): + status, _, errors = self.run_report(implementations=["nothing"]) + self.assertEqual(status, 0) + self.assertIn("None of the implementations has a workflow", errors) + + def test_manual_mode_does_not_wait(self): + self.github.add_run("mygo", memory_text()) + self.github.add_run("tauri", status="in_progress", conclusion=None) + self.run_report() + self.assertIn("Still running: Tauri.", self.body()) + + def test_manual_mode_on_a_closed_pull_request_does_nothing(self): + self.github.pulls[7]["state"] = "closed" + status, _, errors = self.run_report() + self.assertEqual(status, 0) + self.assertIn("is not open", errors) + + def test_dry_run_prints_and_writes_nothing(self): + self.github.add_run("mygo", memory_text()) + _, output, _ = self.run_report(dry_run=True) + self.assertTrue(output.startswith(report.MARKER)) + self.assertEqual(self.github.writes, []) + + def test_a_new_implementation_has_no_baseline_and_is_still_reported(self): + self.github.artifacts = [a for a in self.github.artifacts if a["name"] != "memory-uno"] + run = self.github.add_run("uno", memory_text()) + self.run_report(run_id=run, pull_request=None) + self.assertIn("Uno (no baseline)", self.body()) + + def test_writes_the_step_summary(self): + self.github.add_run("mygo", memory_text()) + with tempfile.TemporaryDirectory() as directory: + summary = Path(directory) / "summary.md" + with mock.patch.dict("os.environ", {"GITHUB_STEP_SUMMARY": str(summary)}): + self.run_report() + text = summary.read_text(encoding="utf-8") + self.assertTrue(text.startswith("### Memory use")) + self.assertNotIn(report.MARKER, text) + + +class GitHubTest(unittest.TestCase): + def completed(self, returncode=0, stdout=b"{}", stderr=b""): + return subprocess.CompletedProcess([], returncode, stdout, stderr) + + def test_builds_the_endpoint_of_the_repository(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed()) as run: + github.get("actions/runs", head_sha="abc", per_page=100) + endpoint = f"repos/{REPOSITORY}/actions/runs?head_sha=abc&per_page=100" + self.assertEqual(run.call_args.args[0], ["gh", "api", "--method", "GET", endpoint]) + + def test_retries_a_failure_of_the_server(self): + github = report.GitHub(REPOSITORY) + answers = [self.completed(1, b"", b"gh: HTTP 502"), self.completed(0, b'{"a": 1}')] + with mock.patch("subprocess.run", side_effect=answers), mock.patch("time.sleep") as sleep: + self.assertEqual(github.get("x"), {"a": 1}) + sleep.assert_called_once() + + def test_does_not_retry_a_refusal(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed(1, b"", b"gh: Not Found (HTTP 404)")) as run: + with self.assertRaises(report.ApiError): + github.get("x") + self.assertEqual(run.call_count, 1) + + def test_gives_up_after_three_attempts(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed(1, b"", b"gh: HTTP 503")) as run: + with mock.patch("time.sleep"), self.assertRaises(report.ApiError): + github.get("x") + self.assertEqual(run.call_count, 3) + + def test_creating_a_comment_is_tried_once(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed(1, b"", b"gh: HTTP 502")) as run: + with mock.patch("time.sleep"), self.assertRaises(report.ApiError): + github.send("POST", "issues/7/comments", {"body": "x"}) + self.assertEqual(run.call_count, 1) + + def test_sends_the_body_on_standard_input_and_accepts_an_empty_answer(self): + github = report.GitHub(REPOSITORY) + with mock.patch("subprocess.run", return_value=self.completed(0, b"")) as run: + self.assertIsNone(github.send("DELETE", "issues/comments/1")) + github.send("PATCH", "issues/comments/1", {"body": "text"}) + self.assertEqual(run.call_args.kwargs["input"], b'{"body": "text"}') + self.assertEqual(run.call_args.args[0][-2:], ["--input", "-"]) + + def test_reads_every_page(self): + github = report.GitHub(REPOSITORY) + pages = [list(range(100)), list(range(5))] + with mock.patch.object(report.GitHub, "get", side_effect=pages): + self.assertEqual(len(list(github.pages("issues/7/comments"))), 105) + + +class MainTest(unittest.TestCase): + def run_main(self, *arguments): + with contextlib.redirect_stderr(io.StringIO()): + return report.main(list(arguments)) + + def test_rejects_a_repository_that_is_not_owner_and_name(self): + with self.assertRaises(SystemExit): + self.run_main("--repository", "x;y", "--implementations", "mygo", "--pull-request", "1") + + def test_rejects_an_implementation_that_is_not_a_plain_name(self): + with self.assertRaises(SystemExit): + self.run_main("--repository", REPOSITORY, "--implementations", "../mygo", "--pull-request", "1") + with self.assertRaises(SystemExit): + self.run_main("--repository", REPOSITORY, "--implementations", "", "--pull-request", "1") + + def test_needs_a_run_or_a_pull_request(self): + with self.assertRaises(SystemExit): + self.run_main("--repository", REPOSITORY, "--implementations", "mygo") + + def test_reports_a_failure_of_the_api_as_an_exit_code(self): + with mock.patch.object(report, "report", side_effect=report.ApiError("boom")): + status = self.run_main( + "--repository", REPOSITORY, "--implementations", "mygo", "--pull-request", "1", "--docs", "missing.md" + ) + self.assertEqual(status, 1) + + def test_the_warning_is_one_line_of_a_workflow_command(self): + error = report.ApiError("POST x: gh: Resource not accessible by integration (HTTP 403)\nsecond line 100%") + output = io.StringIO() + with mock.patch.object(report, "report", side_effect=error), contextlib.redirect_stdout(output): + self.run_main( + "--repository", REPOSITORY, "--implementations", "mygo", "--pull-request", "1", "--docs", "missing.md" + ) + self.assertEqual(len(output.getvalue().splitlines()), 1) + self.assertIn("%0Asecond line 100%25", output.getvalue()) + + def test_a_token_that_cannot_write_is_a_warning_and_not_a_failure(self): + error = report.ApiError("POST issues/7/comments: gh: Resource not accessible by integration (HTTP 403)") + output = io.StringIO() + with mock.patch.object(report, "report", side_effect=error), contextlib.redirect_stdout(output): + status = self.run_main( + "--repository", REPOSITORY, "--implementations", "mygo", "--pull-request", "1", "--docs", "missing.md" + ) + self.assertEqual(status, 0) + self.assertTrue(output.getvalue().startswith("::warning::")) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/capacitor.yml b/.github/workflows/capacitor.yml index 7cfd998..a802fe0 100644 --- a/.github/workflows/capacitor.yml +++ b/.github/workflows/capacitor.yml @@ -118,11 +118,20 @@ jobs: - name: Build the end-to-end test build run: npm run build:e2e - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash ../web/e2e/run.sh "$scenario" "$PWD/node_modules/electron/dist/electron" "$PWD/dist/app-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-capacitor + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Build the release run: | npm run build diff --git a/.github/workflows/compose.yml b/.github/workflows/compose.yml index b596269..b676ec7 100644 --- a/.github/workflows/compose.yml +++ b/.github/workflows/compose.yml @@ -107,6 +107,9 @@ jobs: - name: Build the application image run: gradle --console=plain createDistributable - name: Smoke test the application image and measure its memory + env: + # The measurement writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | binary="$PWD/build/compose/binaries/main/app/vfox-plus/bin/vfox-plus" xvfb-run -a -s "-screen 0 1280x800x24" "$binary" > app.log 2>&1 & @@ -118,6 +121,12 @@ jobs: fi pkill -f "^$binary" xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- bash ../shared/tools/measure.sh "$binary" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-compose + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Build the .deb and the tarball run: | # Reproducible timestamps in the package metadata. diff --git a/.github/workflows/dioxus.yml b/.github/workflows/dioxus.yml index da28098..e1e5554 100644 --- a/.github/workflows/dioxus.yml +++ b/.github/workflows/dioxus.yml @@ -125,11 +125,20 @@ jobs: cargo build --release --locked --features e2e cp target/release/vfox-plus "$RUNNER_TEMP/vfox-plus-e2e" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-dioxus + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Build the release working-directory: dioxus run: cargo build --release --locked diff --git a/.github/workflows/flet.yml b/.github/workflows/flet.yml index 9bd337c..35c7686 100644 --- a/.github/workflows/flet.yml +++ b/.github/workflows/flet.yml @@ -126,12 +126,21 @@ jobs: done - name: Run the end-to-end scenarios in a virtual display working-directory: . + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ env VFOX_PLUS_E2E_TIMEOUT=300 VFOX_PLUS_E2E_VFOX="$VFOX_PLUS_E2E_VFOX" \ bash web/e2e/run.sh "$scenario" "$PWD/flet/build/linux/vfox-plus" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-flet + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Smoke test the release run: | binary="$PWD/build/linux/vfox-plus" diff --git a/.github/workflows/flutter.yml b/.github/workflows/flutter.yml index 835416f..64ff8f0 100644 --- a/.github/workflows/flutter.yml +++ b/.github/workflows/flutter.yml @@ -193,6 +193,13 @@ jobs: run: | xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ python3 tool/measure.py --output "$RUNNER_TEMP/measure" build/linux/x64/release/bundle/vfox-plus + # The numbers, for memory.yml. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-flutter + path: ${{ runner.temp }}/measure/memory.txt + if-no-files-found: error + overwrite: true - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: failure() with: diff --git a/.github/workflows/gio.yml b/.github/workflows/gio.yml index 03c4199..8307b4a 100644 --- a/.github/workflows/gio.yml +++ b/.github/workflows/gio.yml @@ -122,12 +122,21 @@ jobs: tar -xzf "$RUNNER_TEMP/$archive.tar.gz" -C "$RUNNER_TEMP" echo "VFOX_PLUS_E2E_VFOX=$RUNNER_TEMP/$archive/vfox" >> "$GITHUB_ENV" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | bash gio/tools/build.sh --e2e "$RUNNER_TEMP/vfox-plus-e2e" for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-gio + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Smoke test the release run: | binary="$PWD/gio/build/vfox-plus" diff --git a/.github/workflows/memory.yml b/.github/workflows/memory.yml new file mode 100644 index 0000000..3a73ac9 --- /dev/null +++ b/.github/workflows/memory.yml @@ -0,0 +1,86 @@ +# Copyright 2026 Han Li and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The memory of the implementations in a pull request. The workflow of each +# implementation measures its application and uploads the result as the +# artifact memory-. When the last of those workflows has ended +# for the latest commit of a pull request, this one writes a single comment on +# it: a table like the one of docs/memory.md with the difference from the latest +# numbers of main. Later runs replace that comment instead of adding another one +# (.github/scripts/memory_report.py; docs/memory.md#memory-in-pull-requests). +# +# It runs after the workflows of the pull request, in the context of the default +# branch, so that it can write to pull requests of forks, which a pull_request +# workflow cannot. For the same reason it never runs code of the pull request: it +# checks out main, and the artifacts it downloads are data, of which the script +# only reads numbers. +# +# Run it by hand to write the report of a pull request again: +# gh workflow run memory.yml -f pull_request= +name: Memory + +on: + workflow_run: + # The workflows that measure an implementation, by name. Keep this in sync + # with MEMORY_IMPLEMENTATIONS below, which has their file names. + workflows: [Capacitor, Compose, Dioxus, Flet, Flutter, Gio, mygo, Tauri, Uno, Wails] + types: [completed] + workflow_dispatch: + inputs: + pull_request: + description: The number of the pull request to write the report of now + required: true + type: number + +permissions: + actions: read + contents: read + pull-requests: write + +concurrency: + # The workflows of a commit end one after another, and each starts a report. + # They run one at a time, so that two of them cannot both write the comment, + # and the last one to start is the one that finds all of them ended. + group: memory-${{ github.event.workflow_run.head_sha || format('pull-request-{0}', inputs.pull_request) }} + cancel-in-progress: false + +env: + # The file names of the workflows of the implementations (without .yml), which + # are also the names of their directories and of their artifacts. + MEMORY_IMPLEMENTATIONS: capacitor compose dioxus flet flutter gio mygo tauri uno wails + +jobs: + report: + name: Write the report + # Forks do not run CI. + if: >- + github.repository == 'version-fox/vfox-plus' && + (github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'pull_request') + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Write the report + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + PULL_REQUEST: ${{ inputs.pull_request }} + run: | + if [[ -n "$RUN_ID" ]]; then + python3 .github/scripts/memory_report.py --run "$RUN_ID" + else + python3 .github/scripts/memory_report.py --pull-request "$PULL_REQUEST" + fi diff --git a/.github/workflows/mygo.yml b/.github/workflows/mygo.yml index fc2452e..0d1044b 100644 --- a/.github/workflows/mygo.yml +++ b/.github/workflows/mygo.yml @@ -122,12 +122,21 @@ jobs: tar -xzf "$RUNNER_TEMP/$archive.tar.gz" -C "$RUNNER_TEMP" echo "VFOX_PLUS_E2E_VFOX=$RUNNER_TEMP/$archive/vfox" >> "$GITHUB_ENV" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | bash mygo/tools/build.sh --e2e "$RUNNER_TEMP/vfox-plus-e2e" for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-mygo + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Smoke test the release run: | binary="$PWD/mygo/build/vfox-plus" diff --git a/.github/workflows/repository.yml b/.github/workflows/repository.yml index b93993f..165034b 100644 --- a/.github/workflows/repository.yml +++ b/.github/workflows/repository.yml @@ -78,3 +78,6 @@ jobs: for file in shared/icons.json shared/i18n/*.arb shared/fixtures/*/cases.json; do jq empty "$file" || { echo "::error file=$file::Invalid JSON"; exit 1; } done + - name: Test the memory report + # What memory.yml writes on pull requests. + run: python3 -m unittest discover -s .github/scripts -p 'test_*.py' diff --git a/.github/workflows/tauri.yml b/.github/workflows/tauri.yml index e0f746f..601ec06 100644 --- a/.github/workflows/tauri.yml +++ b/.github/workflows/tauri.yml @@ -115,11 +115,20 @@ jobs: '{"build": {"frontendDist": "../../web/dist/tauri-e2e", "beforeBuildCommand": ""}}' cp src-tauri/target/release/vfox-plus "$RUNNER_TEMP/vfox-plus-e2e" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-tauri + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Build the release working-directory: tauri run: npx tauri build --no-bundle diff --git a/.github/workflows/uno.yml b/.github/workflows/uno.yml index ed41f91..551e471 100644 --- a/.github/workflows/uno.yml +++ b/.github/workflows/uno.yml @@ -137,12 +137,21 @@ jobs: working-directory: uno run: dotnet restore src/VfoxPlus.App --runtime linux-x64 -p:Configuration=Release - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | bash uno/tools/publish.sh --e2e "$RUNNER_TEMP/vfox-plus-e2e" for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e/VfoxPlus" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-uno + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Publish the release run: bash uno/tools/publish.sh uno/build/stage - name: Build the .deb and the tarball diff --git a/.github/workflows/wails.yml b/.github/workflows/wails.yml index 7baebe4..319a371 100644 --- a/.github/workflows/wails.yml +++ b/.github/workflows/wails.yml @@ -120,12 +120,21 @@ jobs: tar -xzf "$RUNNER_TEMP/$archive.tar.gz" -C "$RUNNER_TEMP" echo "VFOX_PLUS_E2E_VFOX=$RUNNER_TEMP/$archive/vfox" >> "$GITHUB_ENV" - name: Run the end-to-end scenarios in a virtual display + env: + # The "measure" scenario writes its numbers here for memory.yml. + VFOX_PLUS_MEASURE_OUTPUT: ${{ runner.temp }}/memory/memory.txt run: | bash wails/build.sh --e2e "$RUNNER_TEMP/vfox-plus-e2e" for scenario in fake real measure; do xvfb-run -a -s "-screen 0 1280x800x24" dbus-run-session -- \ bash web/e2e/run.sh "$scenario" "$RUNNER_TEMP/vfox-plus-e2e" done + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: memory-wails + path: ${{ runner.temp }}/memory/memory.txt + if-no-files-found: error + overwrite: true - name: Smoke test the release run: | binary="$PWD/wails/build/vfox-plus" diff --git a/.gitignore b/.gitignore index 2c32fd6..d6ae8b1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,8 @@ Thumbs.db # Packaging output (see flutter/packaging/linux/build_deb.sh). /dist/ + +# Python bytecode, which running the tests of .github/scripts leaves behind. CI +# rejects binary files. +__pycache__/ +*.pyc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b758eb..f71bfe3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,8 @@ all of them. | Workflow | Runs on | Checks | | --- | --- | --- | -| [`repository.yml`](.github/workflows/repository.yml) | every change | License headers (SkyWalking Eyes), workflows (actionlint), shell scripts (ShellCheck), shared JSON, no committed binary files | +| [`repository.yml`](.github/workflows/repository.yml) | every change | License headers (SkyWalking Eyes), workflows (actionlint), shell scripts (ShellCheck), shared JSON, no committed binary files, the tests of the memory report | +| [`memory.yml`](.github/workflows/memory.yml) | the end of the workflows of the implementations, on pull requests | One comment on the pull request with the memory of every implementation, sorted like [docs/memory.md](docs/memory.md#results), and the difference from `main` (see [Memory in pull requests](docs/memory.md#memory-in-pull-requests)) | | [`flutter.yml`](.github/workflows/flutter.yml) | `flutter/`, `shared/` | Lock file, formatting, generated translations, analysis, tests with an 85 % coverage gate, end-to-end tests on Linux under Xvfb against captured output and against the real vfox, release build with a smoke test and a memory measurement, `.deb` and `.tar.gz` packages validated with lintian | | [`flutter-sandbox.yml`](.github/workflows/flutter-sandbox.yml) | `flutter/sandbox/`, `flutter/versions.env` | Builds the Docker sandbox and runs `sandbox.sh all` in it | | [`web.yml`](.github/workflows/web.yml) | `web/`, `shared/` | Formatting (Prettier), lint (ESLint), type checking, unit tests with an 85 % coverage gate, the production builds | @@ -71,6 +72,12 @@ all of them. Dependabot keeps the pub, npm, cargo, Go, Gradle and NuGet dependencies and the GitHub Actions up to date; actions are pinned to commit SHAs. +A pull request that rebuilds an implementation gets a comment with the memory of all of +them and the difference from `main`, which the next commit replaces instead of adding +another one. A change that is more than the noise between CI runs is in bold; read +[Memory in pull requests](docs/memory.md#memory-in-pull-requests) before you trust a small +one. + ## Common changes ### Texts diff --git a/docs/memory.md b/docs/memory.md index 27656fb..7f0d975 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -146,6 +146,78 @@ What is different from the other implementations: - The memory is read when the screen has been still for 3 seconds, not when the application reports a page. +## Memory in pull requests + +A pull request that rebuilds an implementation gets one comment with the memory of all +of them: a table like [the one above](#results), sorted by the PSS of the Home page, +with the difference from `main` next to the PSS. A new commit replaces that comment +instead of adding another one. + +- The workflow of each implementation runs the `measure` scenario of [How it is + measured](#how-it-is-measured) (Flutter: `tool/measure.py`) and uploads the lines that + it prints as the artifact `memory-`. `measure.sh` writes them to the file + named in `VFOX_PLUS_MEASURE_OUTPUT` as well. +- [`memory.yml`](../.github/workflows/memory.yml) starts every time one of those workflows + ends and runs [`memory_report.py`](../.github/scripts/memory_report.py). While another + one is still running for the latest commit of the pull request, the script stops. The + run after the last one compares the artifacts of the pull request with those of the + latest run of the same workflow on `main` and writes the comment. The workflow runs in + the context of `main`, which is what lets it comment on pull requests of forks, and it + reads nothing but two lines of numbers from the artifacts. If its token cannot write the + comment, the run only warns, and the report is in its summary. +- An implementation that the pull request did not rebuild, because its workflow only runs + for changes in its own paths, shows the numbers of `main` and is marked "(main)". +- The numbers are not those of the table above. They come from the runners of GitHub and + from one run, not from Docker on a WSL2 host. Compare the rows of one comment with each + other and with `main`, not with this page. + +### The noise + +The same code does not use the same memory in every CI run. This is the largest +difference between runs of identical code, from the 107 successful runs of 2026-10-18 and +19 whose source was the same as that of at least one other run (30 groups of 2 to 8 runs), +in MB and as a part of the lowest number: + +| Implementation | Home PSS | Versions PSS | +| -------------- | ---------: | -----------: | +| mygo | 1 (2.4%) | 0 (0.0%) | +| Uno | 4 (3.3%) | 5 (3.0%) | +| Gio | 20 (10.4%) | 23 (11.4%) | +| Dioxus | 5 (1.8%) | 13 (4.3%) | +| Compose | 15 (6.1%) | 14 (5.4%) | +| Tauri | 14 (3.3%) | 14 (2.9%) | +| Capacitor | 4 (1.0%) | 3 (0.7%) | +| Flutter | 35 (10.8%) | 43 (13.4%) | +| Flet | 16 (5.2%) | 25 (7.5%) | +| Wails | 11 (2.3%) | 13 (2.4%) | + +So the comment calls a difference a change, and prints it in bold, only when it is more +than 5 MB and more than 5% of the number on `main`; for Compose and Flet more than 8%, for +Gio more than 12% and for Flutter more than 15%, because they vary more. Of the 940 +comparisons of two runs of the same code (the Home and the Versions PSS of every pair, in +both directions), none was a change by these limits. The number of processes never varied, +so a different number is a change too. The sample is small: if the comment reports a +change that the commit cannot have caused, measure how much identical code varies again +and adjust `TOLERANCES` in the script. + +### Using it + +- The artifacts expire after 90 days. An implementation without a run on `main` for that + long has "no baseline" until its workflow runs on `main` again. +- `gh workflow run memory.yml -f pull_request=` writes the report of an open pull + request again. To only see it, run the script with `--dry-run`: + + ```sh + GH_TOKEN="$(gh auth token)" python3 .github/scripts/memory_report.py --dry-run \ + --repository version-fox/vfox-plus --pull-request \ + --implementations "capacitor compose dioxus flet flutter gio mygo tauri uno wails" + ``` + +- The tests of the script run in the `Repository` workflow: + `python3 -m unittest discover -s .github/scripts`. +- An implementation that is added uploads its numbers as `memory-` like the + others and goes into the two lists of `memory.yml`; one that is removed leaves them. + ## Updating this page When an implementation is added, or its memory changes noticeably, run diff --git a/docs/new-implementation.md b/docs/new-implementation.md index 49fa19b..f197675 100644 --- a/docs/new-implementation.md +++ b/docs/new-implementation.md @@ -68,6 +68,13 @@ state layer, and the UI on top. the directory, `shared/` and the workflow itself, and that builds the installable packages as artifacts. Use [`flutter.yml`](../.github/workflows/flutter.yml) as a model. +- In that workflow, the memory measurement (see below), uploaded as the artifact + `memory-` right after it, and the name of the workflow and + `` in the two lists of [`memory.yml`](../.github/workflows/memory.yml), + so that pull requests get the memory of the implementation in a comment (see + [the memory page](memory.md#memory-in-pull-requests)). `VFOX_PLUS_MEASURE_OUTPUT` + makes `measure.sh` write the numbers to the file that the artifact holds; copy the + steps of [`mygo.yml`](../.github/workflows/mygo.yml). ## 5. Package diff --git a/shared/tools/measure.sh b/shared/tools/measure.sh index 5ac0c88..e361b71 100755 --- a/shared/tools/measure.sh +++ b/shared/tools/measure.sh @@ -28,6 +28,10 @@ # "e2e: scenario measure passed" and exit with code 0, waiting a few seconds # after each point so that the measurement sees a settled application. Prints # lines "memory (): RSS MB, PSS MB in processes". +# +# With VFOX_PLUS_MEASURE_OUTPUT set to a file name, the same lines also replace +# the content of that file. CI uploads it for the memory report of pull +# requests (.github/workflows/memory.yml). set -euo pipefail TOOLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -51,6 +55,12 @@ cleanup() { } trap cleanup EXIT +OUTPUT="${VFOX_PLUS_MEASURE_OUTPUT:-}" +if [[ -n "$OUTPUT" ]]; then + mkdir -p "$(dirname "$OUTPUT")" + : > "$OUTPUT" +fi + mkdir -p "$HOME_DIR/.config/vfox-plus" vfox="$SANDBOX/vfox" install -m 755 "$SHARED/fixtures/fake-vfox.sh" "$vfox" @@ -85,13 +95,15 @@ APP_PID=$! printed=0 deadline=$((SECONDS + ${VFOX_PLUS_E2E_TIMEOUT:-300})) show_new_lines() { - local lines + local lines result lines="$(wc -l < "$SANDBOX/app.log")" if [[ $lines -gt $printed ]]; then while IFS= read -r line; do echo "$line" if [[ "$line" == *"e2e: MEASURE "* ]]; then - echo "memory (${line##*MEASURE }): $(memory_of "$APP_PID")" + result="memory (${line##*MEASURE }): $(memory_of "$APP_PID")" + echo "$result" + [[ -z "$OUTPUT" ]] || echo "$result" >> "$OUTPUT" fi done < <(sed -n "$((printed + 1)),${lines}p" "$SANDBOX/app.log") printed=$lines