From 449837a5ec0eb6f9515bf1389160521e06576296 Mon Sep 17 00:00:00 2001 From: Ryan Barlow <7389646+ryanbarlow97@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:34:43 +0000 Subject: [PATCH 1/3] fix(servers): check releases fresh without response caches --- backend/src/servers/control.py | 23 +- backend/src/servers/host.py | 22 +- backend/src/servers/inventory.py | 611 +++++++++------ backend/src/servers/test_control.py | 133 ++++ backend/src/servers/test_inventory.py | 735 ++++++++++++------ docker-compose.staging.yml | 1 + .../components/admin/ServersOverview.test.tsx | 68 +- .../app/components/admin/ServersOverview.tsx | 87 ++- frontend/lib/admin/servers.test.ts | 6 + frontend/lib/admin/servers.ts | 11 +- 10 files changed, 1166 insertions(+), 531 deletions(-) diff --git a/backend/src/servers/control.py b/backend/src/servers/control.py index d2e6c533..13f76539 100644 --- a/backend/src/servers/control.py +++ b/backend/src/servers/control.py @@ -4,6 +4,7 @@ import json import os import socket +from pathlib import Path from src.servers import inventory @@ -56,13 +57,27 @@ def plan(server: str, mode: str) -> dict: if mode == "latest": if inventory.github_org() != inventory.DEFAULT_ORG: raise ControlError("release_org_not_supported", 400) - # A write must never silently use stale release information. - releases, _, failed = inventory.releases_cache.get(wait=True) - if releases is None or failed: + variable = "SERVER_MAIN_DIR" if server == "main" else "SERVER_DEV_DIR" + root = os.getenv(variable, "").strip() + if not root or not (Path(root) / "plugins").is_dir(): + raise ControlError("server_files_unavailable", 503) + releases, servers = inventory.check_servers({server: (Path(root), server == "main")}) + if releases.error or releases.errors: raise ControlError("releases_unavailable", 503) + plugins, others = inventory.compare(releases, servers) + if any(p["servers"][server]["state"] == "unverified" for p in plugins) or any(o.get("unverified") for o in others): + raise ControlError("releases_unavailable", 503) + expected = [{"file": ("update/" if j.update else "") + j.file, "sha256": j.sha256, + "name": j.name, "version": j.version} for j in servers[server]["jars"]] for repo, found in releases.items(): release = found[0] candidates.append({"repo": repo, "tag": release.tag, "asset": release.asset, "sha256": release.sha256, "known": [{"sha256": r.sha256, "version": r.version} for r in found]}) - return request("plan", server=server, mode=mode, releases=candidates) + try: + return request("plan", server=server, mode=mode, releases=candidates, + **({"expected": expected} if mode == "latest" else {})) + except ControlError as error: + if mode == "latest" and error.code == "bad_control_request": + raise ControlError("control_unavailable", 503) from None + raise diff --git a/backend/src/servers/host.py b/backend/src/servers/host.py index 59099639..434c03de 100644 --- a/backend/src/servers/host.py +++ b/backend/src/servers/host.py @@ -156,14 +156,14 @@ def snapshot(self, server: str) -> list[dict]: if path.is_symlink() or not path.is_file(): raise ControlError("unsafe_plugin_file") try: - _, name, version = inventory._describe(path) + sha, name, version = inventory._describe(path) if not name or not version or name.lower() in names: raise ControlError("plugin_inventory_conflict") if prefix: raise ControlError("pending_plugin_update") names.add(name.lower()) - # Always hash afresh before a write, even for unchanged size/mtime. - result.append({"file": path.name, "sha256": digest(path), "name": name, "version": version}) + # _describe reads fresh bytes and metadata from the same open file. + result.append({"file": path.name, "sha256": sha, "name": name, "version": version}) except PermissionError: raise ControlError("plugin_unreadable") from None return result @@ -181,7 +181,7 @@ def _idle(self, server): for key, job in self.jobs.items()): raise ControlError("server_job_active") - def plan(self, server: str, mode: str, releases: list[dict]) -> dict: + def plan(self, server: str, mode: str, releases: list[dict], expected: list[dict] | None = None) -> dict: if mode not in {"restart", "latest"} or not isinstance(releases, list) or len(releases) > 100: raise ControlError("bad_server_action", 400) with self.lock: @@ -194,6 +194,8 @@ def plan(self, server: str, mode: str, releases: list[dict]) -> dict: if not (folder / "job.json").exists() and (not path.exists() or json.loads(path.read_text())["expires_at"] < time.time()): shutil.rmtree(folder) baseline = self.snapshot(server) + if expected is not None and baseline != expected: + raise ControlError("plugins_changed") _, logs = self._paths(server) startup = inventory.read_startup(logs) if not startup or not startup["done"] or self._maintenance(server): @@ -216,7 +218,7 @@ def plan(self, server: str, mode: str, releases: list[dict]) -> dict: version = release["tag"].removeprefix("v") prefix = release["asset"].removesuffix(".jar").removesuffix("-" + version) old = [jar for jar in baseline if jar["file"].lower().startswith(prefix.lower() + "-") - or jar["name"].lower() == release["repo"].lower()] + or inventory.plugin_identifier(jar["name"]) == release["repo"].lower()] if not old: continue # Never add plugins that are not already installed. if len(old) != 1 or old[0]["file"] in used: @@ -630,15 +632,19 @@ def dispatch(self, body: dict) -> dict: "cancel": {"operation", "server", "job_id", "actor", "reason"}, "reopen": {"operation", "server", "job_id", "actor", "reason"}, "resolve": {"operation", "server", "job_id", "actor", "reason"}} - # Older websites omit the optional test-build list. - if operation not in allowed or set(body) - ({"replace_test_builds"} if operation == "start" else set()) != allowed[operation]: + # Older websites omit the optional baseline and test-build choices. + optional = {"replace_test_builds"} if operation == "start" else {"expected"} if operation == "plan" else set() + if operation not in allowed or set(body) - optional != allowed[operation]: raise ControlError("bad_control_request", 400) if operation == "status": return {"jobs": json.loads(self.status_json)} server = body["server"] self._paths(server) if operation == "plan": - return self.plan(server, body["mode"], body["releases"]) + expected = body.get("expected") + if "expected" in body and (not isinstance(expected, list) or len(expected) > 300): + raise ControlError("bad_control_request", 400) + return self.plan(server, body["mode"], body["releases"], expected) if any(not isinstance(body[key], str) or not 3 <= len(body[key]) <= 500 for key in ("actor", "reason")): raise ControlError("bad_control_request", 400) if operation == "start": diff --git a/backend/src/servers/inventory.py b/backend/src/servers/inventory.py index 499456ec..94db3ce1 100644 --- a/backend/src/servers/inventory.py +++ b/backend/src/servers/inventory.py @@ -37,13 +37,6 @@ ("dev", "Dev", "SERVER_DEV_DIR", False), ) DEFAULT_ORG = "TF-Minecraft" -# Releases are fetched again after this many seconds; failures are retried sooner. -# With a token a check is about 13 GraphQL queries; without one it is about 40 REST -# requests, and GitHub allows 60 an hour. -RELEASES_TTL = 600 -RELEASES_TTL_WITHOUT_TOKEN = 3600 -RETRY_TTL = 60 -RELEASES_PER_REPO = 30 # Older rotated logs are read until one holds a startup. MAX_LOGS = 12 @@ -85,6 +78,24 @@ def prefix(self) -> str: # GitHub # -------------------- +# Each check owns its results. The slots bound work, never share responses. +CHECK_TIMEOUT = 20 +GRAPHQL_CHUNK = 8 +GITHUB_WORKERS = 6 +RELEASES_PER_REPO = 30 +_check_slots = threading.BoundedSemaphore(2) + + +class Releases(dict): + def __init__(self): + super().__init__() + self.repos: set[str] = set() + self.errors: dict[str, str] = {} + self.complete: set[str] = set() + self.error: str | None = None + self.checked_at: str | None = None + + def github_org() -> str: return os.getenv("SERVERS_GITHUB_ORG", "").strip() or DEFAULT_ORG @@ -93,243 +104,340 @@ def _token() -> str: return os.getenv("SERVERS_GITHUB_TOKEN", "").strip() or os.getenv("PATCHNOTES_GITHUB_TOKEN", "").strip() -def _github(path: str, body: dict | None = None) -> Any: - """A REST GET, or a POST of `body` (a GraphQL query to `/graphql`).""" - headers = {"User-Agent": "TFMC-Servers/1.0", "Accept": "application/vnd.github+json"} - token = _token() - if token: +def _github(path: str, body: dict | None = None, *, timeout: float = CHECK_TIMEOUT) -> Any: + headers = {"User-Agent": "TFMC-Servers/1.0", "Accept": "application/vnd.github+json", "Cache-Control": "no-cache"} + if token := _token(): headers["Authorization"] = f"Bearer {token}" data = None if body is None else json.dumps(body).encode() request = urllib.request.Request(f"https://api.github.com{path}", data=data, headers=headers) - with urllib.request.urlopen(request, timeout=20) as response: - return json.loads(response.read().decode() or "null") + deadline = time.monotonic() + timeout + with urllib.request.urlopen(request, timeout=timeout) as response: + chunks, size = [], 0 + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("GitHub check deadline") + sock = getattr(getattr(getattr(response, "fp", None), "raw", None), "_sock", None) + if sock is not None: + sock.settimeout(remaining) + chunk = response.read1(65536) + if time.monotonic() >= deadline: + raise TimeoutError("GitHub check deadline") + if not chunk: + break + size += len(chunk) + if size > 8 << 20: + raise ValueError("GitHub response too large") + chunks.append(chunk) + return json.loads(b"".join(chunks).decode() or "null") + + +def _fetcher(fetch: Fetch | None, deadline: float) -> Fetch: + def request(path: str, body: dict | None = None): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("GitHub check deadline") + result = fetch(path, body) if fetch else _github(path, body, timeout=remaining) + if time.monotonic() >= deadline: + raise TimeoutError("GitHub check deadline") + return result + return request def repo_releases(repo: str, data: Any) -> list[Release]: - """A repository's published plugin releases, newest first. Drafts and pre-releases don't count.""" found = [] for release in data if isinstance(data, list) else []: if not isinstance(release, dict) or release.get("draft") or release.get("prerelease"): continue - published = release.get("published_at") - tag = str(release.get("tag_name") or "") - jar = next((a for a in release.get("assets") or [] - if isinstance(a, dict) and str(a.get("name", "")).lower().endswith(".jar") - and str(a.get("digest") or "").startswith("sha256:")), None) - if not published or not tag or jar is None: + published, tag = release.get("published_at"), str(release.get("tag_name") or "") + if not published or not tag: continue - found.append(Release(repo=repo, version=tag[1:] if tag[:1] in "vV" else tag, published_at=published, - url=str(release.get("html_url") or ""), asset=str(jar["name"]), - sha256=str(jar["digest"]).split(":", 1)[1].lower(), tag=tag)) + for jar in release.get("assets") or []: + if not isinstance(jar, dict) or not str(jar.get("name", "")).lower().endswith(".jar"): + continue + digest = str(jar.get("digest") or "") + if not digest.startswith("sha256:") or not digest[7:]: + continue + found.append(Release(repo=repo, version=tag[1:] if tag[:1] in "vV" else tag, published_at=published, + url=str(release.get("html_url") or ""), asset=str(jar["name"]), + sha256=digest[7:].lower(), tag=tag)) return sorted(found, key=lambda r: r.published_at, reverse=True) -def fetch_releases(fetch: Fetch | None = None) -> dict[str, list[Release]]: - """Every active repository's published plugin releases, by repository name. +class HistoryPage(list): + def __init__(self, repo: str, rows: list[dict]): + super().__init__(repo_releases(repo, rows)) + # Missing asset digests prevent a negative proof even when every page was read. + self.verifiable = not any( + isinstance(r, dict) and r.get("published_at") and not r.get("draft") and not r.get("prerelease") + and any(isinstance(a, dict) and str(a.get("name", "")).lower().endswith(".jar") + and (not str(a.get("digest") or "").startswith("sha256:") or not str(a.get("digest") or "")[7:]) + for a in r.get("assets") or []) + for r in rows) - With a token this takes a handful of GraphQL queries, and each latest release - is compared with its default branch. Without one GraphQL isn't available, so it - reads each repository's releases over REST and skips the comparison. - """ - fetch = fetch or _github - if _token(): - return _graphql_releases(fetch) - org = urllib.parse.quote(github_org()) - repos: list[str] = [] - for page in range(1, 4): - data = fetch(f"/orgs/{org}/repos?per_page=100&page={page}") - if not isinstance(data, list): - raise ValueError("unexpected repository list") - repos += [str(r["name"]) for r in data if isinstance(r, dict) and r.get("name") and not r.get("archived")] - if len(data) < 100: - break - - def one(repo: str) -> tuple[str, list[Release]]: - data = fetch(f"/repos/{org}/{urllib.parse.quote(repo)}/releases?per_page={RELEASES_PER_REPO}") - return repo, repo_releases(repo, data) - - with ThreadPoolExecutor(max_workers=8) as pool: - return {repo: found for repo, found in pool.map(one, repos) if found} - - -# Repositories per releases (and comparison) query: GitHub answers several small -# queries in parallel much sooner than one large one. -GRAPHQL_CHUNK = 8 _REPOSITORIES = """query($org: String!, $after: String) { - organization(login: $org) { - repositories(first: 100, after: $after, isArchived: false) { - pageInfo { hasNextPage endCursor } - nodes { name defaultBranchRef { name } } - } - } + organization(login: $org) { repositories(first: 100, after: $after, isArchived: false) { + pageInfo { hasNextPage endCursor } nodes { name } + } } }""" -_RELEASES = """fragment releases on Repository { - releases(first: %d, orderBy: {field: CREATED_AT, direction: DESC}) { - nodes { tagName publishedAt isDraft isPrerelease url releaseAssets(first: 20) { nodes { name digest } } } - } -}""" % RELEASES_PER_REPO +_ASSETS = "releaseAssets(first: 20) { pageInfo { hasNextPage endCursor } nodes { name digest } }" +_RELEASE_FIELDS = "tagName publishedAt isDraft isPrerelease url " + _ASSETS +_LATEST = "name defaultBranchRef { name target { oid } } latestRelease { tagCommit { oid } " + _RELEASE_FIELDS + " }" def _graphql(fetch: Fetch, query: str, variables: dict, partial: bool = False) -> dict: - """A query's data. GitHub answers errors in some fields with the rest of the data; - that fails the query unless `partial` allows it.""" - data = fetch("/graphql", {"query": query, "variables": variables}) - if not isinstance(data, dict) or not isinstance(data.get("data"), dict) or (data.get("errors") and not partial): + raw = fetch("/graphql", {"query": query, "variables": variables}) + if not isinstance(raw, dict) or not isinstance(raw.get("data"), dict) or (raw.get("errors") and not partial): raise ValueError("unexpected GraphQL answer") - return data["data"] + data = raw["data"] + if partial and raw.get("errors"): + data = {**data, "_errors": raw["errors"]} + return data -def _aliased(repos: list[str], body: Callable[[int], str], extra: Callable[[int, str], dict] | None = None - ) -> tuple[str, dict]: - """One `rN: repository(...)` field per repository, with its name passed as a variable.""" - fields, params, variables = [], ["$org: String!"], {"org": github_org()} +def _aliased(repos: list[str], body: Callable[[int], str], extra: Callable[[int, str], dict] | None = None): + params, variables, fields = ["$org: String!"], {"org": github_org()}, [] for i, repo in enumerate(repos): - params.append(f"$r{i}: String!") - variables[f"r{i}"] = repo + params.append(f"$r{i}: String!"); variables[f"r{i}"] = repo for name, value in (extra(i, repo) if extra else {}).items(): - params.append(f"${name}: String!") - variables[name] = value + params.append(f"${name}: String!"); variables[name] = value fields.append(f"r{i}: repository(owner: $org, name: $r{i}) {{ {body(i)} }}") - return f"query({', '.join(params)}) {{\n " + "\n ".join(fields) + "\n}", variables - - -def _graphql_releases(fetch: Fetch) -> dict[str, list[Release]]: - branches: dict[str, str] = {} - after = None - for _ in range(3): - page = _graphql(fetch, _REPOSITORIES, {"org": github_org(), "after": after})["organization"]["repositories"] - branches.update((str(r["name"]), str((r.get("defaultBranchRef") or {}).get("name") or "main")) - for r in page["nodes"] if isinstance(r, dict) and r.get("name")) - if not page["pageInfo"]["hasNextPage"]: - break - after = page["pageInfo"]["endCursor"] - - def chunk(repos: list[str]) -> dict[str, list[Release]]: - query, variables = _aliased(repos, lambda i: "...releases") - data = _graphql(fetch, f"{query}\n{_RELEASES}", variables) - found = {} - for i, repo in enumerate(repos): - nodes = ((data.get(f"r{i}") or {}).get("releases") or {}).get("nodes") or [] - found[repo] = repo_releases(repo, [{ - "tag_name": n.get("tagName"), "published_at": n.get("publishedAt"), "draft": n.get("isDraft"), - "prerelease": n.get("isPrerelease"), "html_url": n.get("url"), - "assets": (n.get("releaseAssets") or {}).get("nodes") or [], - } for n in nodes if isinstance(n, dict)]) - found = {repo: f for repo, f in found.items() if f} - compared = _compare_branches(fetch, {repo: (f[0].tag, branches[repo]) for repo, f in found.items()}) - for repo, state in compared.items(): - found[repo][0] = replace(found[repo][0], branch=state) - return found - - names = sorted(branches) - releases: dict[str, list[Release]] = {} - with ThreadPoolExecutor(max_workers=6) as pool: - for found in pool.map(chunk, [names[i:i + GRAPHQL_CHUNK] for i in range(0, len(names), GRAPHQL_CHUNK)]): - releases.update(found) - return releases + return f"query({', '.join(params)}) {{ " + " ".join(fields) + " }", variables + + +def _assets(repo: str, node: dict, fetch: Fetch) -> dict: + connection = node.get("releaseAssets") or {} + assets = list(connection.get("nodes") or []) + while (connection.get("pageInfo") or {}).get("hasNextPage"): + cursor = connection["pageInfo"].get("endCursor") + if not cursor: + raise ValueError("missing asset cursor") + query = """query($org:String!,$repo:String!,$tag:String!,$after:String!){repository(owner:$org,name:$repo){ + release(tagName:$tag){releaseAssets(first:20,after:$after){pageInfo{hasNextPage endCursor} nodes{name digest}}}}}""" + raw = _graphql(fetch, query, {"org": github_org(), "repo": repo, "tag": node["tagName"], "after": cursor}) + release = (raw.get("repository") or {}).get("release") + if not release: + raise ValueError("release disappeared") + connection = release["releaseAssets"] + if (connection.get("pageInfo") or {}).get("endCursor") == cursor and connection["pageInfo"].get("hasNextPage"): + raise ValueError("asset cursor did not advance") + assets.extend(connection.get("nodes") or []) + return {"tag_name": node.get("tagName"), "published_at": node.get("publishedAt"), + "draft": node.get("isDraft"), "prerelease": node.get("isPrerelease"), + "html_url": node.get("url"), "assets": assets} def _compare_branches(fetch: Fetch, latest: dict[str, tuple[str, str]]) -> dict[str, dict]: - """Each repository's default branch against its latest release's tag, in one query. - - One large query is much slower than several small ones, so each releases - query is followed by the comparisons for its own repositories. - - A failure leaves the releases without a comparison rather than failing them. - """ - repos = sorted(latest) - if not repos: + if not latest: return {} - query, variables = _aliased( - repos, lambda i: f"ref(qualifiedName: $t{i}) {{ compare(headRef: $b{i}) {{ status aheadBy }} }}", + repos = sorted(latest) + query, variables = _aliased(repos, + lambda i: f"ref(qualifiedName: $t{i}) {{ compare(headRef: $b{i}) {{ status aheadBy }} }}", lambda i, repo: {f"t{i}": f"refs/tags/{latest[repo][0]}", f"b{i}": latest[repo][1]}) try: data = _graphql(fetch, query, variables, partial=True) - except (urllib.error.URLError, http.client.HTTPException, OSError, ValueError): + except (OSError, ValueError, http.client.HTTPException): return {} - org = urllib.parse.quote(github_org()) found = {} for i, repo in enumerate(repos): compare = (((data.get(f"r{i}") or {}).get("ref") or {}).get("compare") or {}) status = str(compare.get("status") or "").lower() - if status not in {"identical", "ahead", "behind", "diverged"}: - continue - tag, branch = latest[repo] - found[repo] = {"name": branch, "status": status, "ahead": int(compare.get("aheadBy") or 0), - "url": f"https://github.com/{org}/{urllib.parse.quote(repo)}/compare/" - f"{urllib.parse.quote(tag, safe='')}...{urllib.parse.quote(branch, safe='')}"} + if status in {"identical", "ahead", "behind", "diverged"}: + tag, branch = latest[repo] + found[repo] = _branch(repo, tag, branch, status, int(compare.get("aheadBy") or 0)) return found -class _ReleaseCache: - """Releases, refreshed after a while. +def _branch(repo, tag, branch, status, ahead=0): + return {"name": branch, "status": status, "ahead": ahead, + "url": f"https://github.com/{urllib.parse.quote(github_org())}/{urllib.parse.quote(repo)}/compare/" + f"{urllib.parse.quote(tag, safe='')}...{urllib.parse.quote(branch, safe='')}"} + + +def _history(repo: str, fetch: Fetch, *, graphql: bool): + """Yield pages until exhausted; a deadline/error must not prove 'not a release'.""" + cursor, page = None, 1 + while True: + if graphql: + query = "query($org:String!,$repo:String!,$after:String){repository(owner:$org,name:$repo){releases(first:30,after:$after,orderBy:{field:CREATED_AT,direction:DESC}){pageInfo{hasNextPage endCursor} nodes{" + _RELEASE_FIELDS + "}}}}" + data = _graphql(fetch, query, {"org": github_org(), "repo": repo, "after": cursor}) + repository = data.get("repository") + if not repository or not isinstance(repository.get("releases"), dict): + raise ValueError("repository disappeared") + connection = repository["releases"] + yield HistoryPage(repo, [_assets(repo, n, fetch) for n in connection.get("nodes") or []]) + info = connection.get("pageInfo") or {} + if not info.get("hasNextPage"): + return + new_cursor = info.get("endCursor") + if not new_cursor or new_cursor == cursor: + raise ValueError("release cursor did not advance") + cursor = new_cursor + else: + rows = fetch(f"/repos/{urllib.parse.quote(github_org())}/{urllib.parse.quote(repo)}/releases?per_page=30&page={page}") + if not isinstance(rows, list): + raise ValueError("unexpected release list") + yield HistoryPage(repo, rows) + if len(rows) < RELEASES_PER_REPO: + return + page += 1 + + +def _latest(fetch: Fetch) -> Releases: + found = Releases() + if not _token(): + for page in range(1, 100): + rows = fetch(f"/orgs/{urllib.parse.quote(github_org())}/repos?per_page=100&page={page}") + if not isinstance(rows, list): + raise ValueError("unexpected repository list") + found.repos.update(r["name"] for r in rows if isinstance(r, dict) and r.get("name") and not r.get("archived")) + if len(rows) < 100: + break + def rest(repo): + try: + try: + data = fetch(f"/repos/{urllib.parse.quote(github_org())}/{urllib.parse.quote(repo)}/releases/latest") + except urllib.error.HTTPError as error: + if error.code != 404: + raise + return repo, [], None + releases = repo_releases(repo, [data]) + if not releases: + releases = next((p for p in _history(repo, fetch, graphql=False) if p), []) + return repo, releases[:1], None + except (OSError, ValueError, http.client.HTTPException): + return repo, [], "github_unavailable" + with ThreadPoolExecutor(max_workers=GITHUB_WORKERS) as pool: + for repo, releases, error in pool.map(rest, sorted(found.repos)): + if releases: found[repo] = releases + if error: found.errors[repo] = error + else: + cursor = None + while True: + data = _graphql(fetch, _REPOSITORIES, {"org": github_org(), "after": cursor}) + connection = (data.get("organization") or {}).get("repositories") + if not isinstance(connection, dict): + raise ValueError("unexpected repository list") + found.repos.update(n["name"] for n in connection.get("nodes") or [] if isinstance(n, dict) and n.get("name")) + info = connection.get("pageInfo") or {} + if not info.get("hasNextPage"): break + next_cursor = info.get("endCursor") + if not next_cursor or next_cursor == cursor: raise ValueError("missing repository cursor") + cursor = next_cursor + def chunk(repos): + local, errors, unequal = {}, {}, {} + try: + query, variables = _aliased(repos, lambda i: _LATEST) + data = _graphql(fetch, query, variables, partial=True) + except (OSError, ValueError, http.client.HTTPException): + return {}, {r: "github_unavailable" for r in repos} + for i, repo in enumerate(repos): + try: + if any((error.get("path") or [None])[0] == f"r{i}" for error in data.get("_errors", [])): + raise ValueError("repository lookup failed") + node = data.get(f"r{i}") + if not isinstance(node, dict) or "latestRelease" not in node: + raise ValueError("missing repository") + latest = node["latestRelease"] + if latest is None: continue + releases = repo_releases(repo, [_assets(repo, latest, fetch)]) + if not releases: + releases = next((p for p in _history(repo, fetch, graphql=True) if p), []) + if not releases: continue + release = releases[0]; branch = node.get("defaultBranchRef") or {} + head = (branch.get("target") or {}).get("oid") + commit = (latest.get("tagCommit") or {}).get("oid") if release.tag == latest.get("tagName") else None + if branch.get("name") and head and commit: + if head == commit: + release = replace(release, branch=_branch(repo, release.tag, branch["name"], "identical")) + else: unequal[repo] = (release.tag, branch["name"]) + local[repo] = [release] + except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException): + errors[repo] = "github_unavailable" + for repo, state in _compare_branches(fetch, unequal).items(): + local[repo][0] = replace(local[repo][0], branch=state) + return local, errors + names = sorted(found.repos) + with ThreadPoolExecutor(max_workers=GITHUB_WORKERS) as pool: + for releases, errors in pool.map(chunk, [names[i:i + GRAPHQL_CHUNK] for i in range(0, len(names), GRAPHQL_CHUNK)]): + found.update(releases); found.errors.update(errors) + found.checked_at = _now() + return found - GitHub is read by one fetch at a time, in a thread of its own. Only the - first fetch makes callers wait; after that an expired cache is refreshed in - the background and callers get the releases already known, unless they ask - to wait for the refresh. - """ - def __init__(self): - self._lock = threading.Lock() - self._releases: dict[str, list[Release]] | None = None - self._checked_at: str | None = None - self._next = 0.0 - self._failed = False - self._fetch: threading.Thread | None = None - # Bumped by clear(), so a fetch running then is ignored. - self._generation = 0 - - def get(self, fetch: Fetch | None = None, wait: bool = False - ) -> tuple[dict[str, list[Release]] | None, str | None, bool]: - """(releases, when they were fetched, whether the last fetch failed). Keeps old releases on failure.""" - with self._lock: - if time.monotonic() < self._next: - return self._state() - if self._fetch is None or not self._fetch.is_alive(): - self._fetch = threading.Thread(target=self._run, args=(fetch, self._generation), daemon=True) - self._fetch.start() - running = self._fetch - if self._releases is not None and not wait: - return self._state() - running.join() - with self._lock: - return self._state() - - def wait_for_refresh(self): - running = self._fetch - if running is not None: - running.join() - - def _state(self) -> tuple[dict[str, list[Release]] | None, str | None, bool]: - return self._releases, self._checked_at, self._failed - - def _run(self, fetch: Fetch | None, generation: int): +def _resolve(found: Releases, jars: list[Jar], fetch: Fetch): + by_hash = {r.sha256: r for rows in found.values() for r in rows} + prefixes = sorted({(r.prefix.lower(), repo) for repo, rows in found.items() for r in rows}, key=lambda x: -len(x[0])) + needed: dict[str, list[Jar]] = {} + for jar in jars: + repo = _match(jar, by_hash, prefixes) + if repo in found and jar.sha256 not in by_hash: + needed.setdefault(repo, []).append(jar) + def resolve(item): + repo, candidates = item; releases = list(found[repo]); complete = False + wanted = {j.sha256 for j in candidates} try: - releases, failed = fetch_releases(fetch), False - except Exception: - # Whatever went wrong, record it: a caller waiting on this fetch must not take the old releases as fresh. - releases, failed = None, True - with self._lock: - if generation != self._generation: - return - now = time.monotonic() - if failed: - self._failed = True - self._next = now + RETRY_TTL + if _token(): + versions = sorted({j.version for j in candidates if j.version and re.fullmatch(r"\d+(?:\.\d+)+(?:-[0-9A-Za-z.-]+)?", j.version)}) + tags = list(dict.fromkeys(t for v in versions for t in ("v" + v, v))) + if tags: + fields = " ".join(f"t{i}:release(tagName:$t{i}){{{_RELEASE_FIELDS}}}" for i in range(len(tags))) + params = ",".join(f"$t{i}:String!" for i in range(len(tags))) + query = f"query($org:String!,$repo:String!,{params}){{repository(owner:$org,name:$repo){{{fields}}}}}" + data = _graphql(fetch, query, {"org": github_org(), "repo": repo, **{f"t{i}": t for i,t in enumerate(tags)}}) + repository = data.get("repository") + if not isinstance(repository, dict): raise ValueError("repository disappeared") + releases.extend(repo_releases(repo, [_assets(repo, n, fetch) for n in repository.values() if n])) + if not wanted <= {r.sha256 for r in releases}: + verifiable = True + for page in _history(repo, fetch, graphql=bool(_token())): + verifiable = verifiable and page.verifiable + releases.extend(page) + if wanted <= {r.sha256 for r in releases}: break + else: complete = verifiable + except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException): + pass # Unknown bytes stay unverified until history has been exhausted. + return repo, list({r.sha256: r for r in reversed(releases)}.values()), complete + with ThreadPoolExecutor(max_workers=GITHUB_WORKERS) as pool: + for repo, releases, complete in pool.map(resolve, needed.items()): + # Keep the designated latest at index zero even after exact/history lookups. + latest = found[repo][0] + found[repo] = [latest] + [r for r in releases if r.sha256 != latest.sha256] + if complete: found.complete.add(repo) + + +def _local_server(root: Path, releases_only: bool): + return {"jars": read_jars(root / "plugins"), "startup": read_startup(root / "logs"), "releases_only": releases_only} + + +def check_servers(roots: dict[str, tuple[Path, bool]], fetch: Fetch | None = None): + """Fresh GitHub and local reads overlap; no response survives this call.""" + deadline = time.monotonic() + CHECK_TIMEOUT + request = _fetcher(fetch, deadline) + acquired = _check_slots.acquire(blocking=False) + found = Releases() + try: + with ThreadPoolExecutor(max_workers=max(1, len(roots) + 1)) as pool: + locals_ = {key: pool.submit(_local_server, root, only) for key, (root, only) in roots.items()} + remote = pool.submit(_latest, request) if acquired else None + servers = {key: future.result() for key, future in locals_.items()} + if remote is None: + found.error = "busy" else: - self._releases, self._checked_at, self._failed = releases, _now(), False - self._next = now + (RELEASES_TTL if _token() else RELEASES_TTL_WITHOUT_TOKEN) - - def clear(self): - with self._lock: - self._releases, self._checked_at, self._next, self._failed = None, None, 0.0, False - self._fetch = None - self._generation += 1 + try: + found = remote.result() + _resolve(found, [j for server in servers.values() for j in server["jars"]], request) + except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException): + found = Releases(); found.error = "github_unavailable" + return found, servers + finally: + if acquired: _check_slots.release() -releases_cache = _ReleaseCache() +def fetch_releases(fetch: Fetch | None = None) -> Releases: + """Fresh latest-release metadata; installed history is resolved by check_servers.""" + return check_servers({}, fetch)[0] # -------------------- @@ -346,35 +454,27 @@ class Jar: update: bool -_hashes: dict[tuple[str, int, int], tuple[str, str | None, str | None]] = {} -_hashes_lock = threading.Lock() +def _archive_metadata(archive) -> tuple[str | None, str | None]: + for entry in ("paper-plugin.yml", "plugin.yml"): + if entry in archive.namelist(): + fields = dict(_YAML_FIELD.findall(archive.read(entry).decode("utf-8", "replace"))) + return fields.get("name") or None, fields.get("version") or None + return None, None def _describe(path: Path) -> tuple[str, str | None, str | None]: - """A jar's SHA-256 and its plugin.yml name and version, cached until the file changes.""" - stat = path.stat() - key = (str(path), stat.st_size, stat.st_mtime_ns) - with _hashes_lock: - if key in _hashes: - return _hashes[key] digest = hashlib.sha256() + name = version = None with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1 << 20), b""): digest.update(chunk) - name = version = None - try: - with zipfile.ZipFile(path) as archive: - for entry in ("paper-plugin.yml", "plugin.yml"): - if entry in archive.namelist(): - fields = dict(_YAML_FIELD.findall(archive.read(entry).decode("utf-8", "replace"))) - name, version = fields.get("name") or None, fields.get("version") or None - break - except (zipfile.BadZipFile, OSError): - pass - value = (digest.hexdigest(), name, version) - with _hashes_lock: - _hashes[key] = value - return value + handle.seek(0) + try: + with zipfile.ZipFile(handle) as archive: + name, version = _archive_metadata(archive) + except zipfile.BadZipFile: + pass + return digest.hexdigest(), name, version def read_jars(plugins: Path) -> list[Jar]: @@ -440,7 +540,7 @@ def read_startup(logs: Path) -> dict | None: # -------------------- def _plugin_state(jars: list[Jar], latest: Release, by_hash: dict[str, Release], startup: dict | None, - releases_only: bool) -> dict: + releases_only: bool, complete: bool = True) -> dict: if not jars: return {"state": "missing", "files": [], "version": None, "release": None, "loaded": None, "loaded_state": "unknown", "update": False, "duplicate": False} @@ -453,7 +553,7 @@ def _plugin_state(jars: list[Jar], latest: Release, by_hash: dict[str, Release], jar = next((j for j in main_jars if loaded is not None and j.version == loaded), main_jars[0]) release = by_hash.get(jar.sha256) if release is None: - state = "not_release" if releases_only else "test_build" + state = ("not_release" if releases_only else "test_build") if complete else "unverified" elif release.version == latest.version: state = "latest" else: @@ -478,11 +578,21 @@ def _plugin_state(jars: list[Jar], latest: Release, by_hash: dict[str, Release], } +def plugin_identifier(name: str | None) -> str: + lowered = (name or "").lower() + return {"activity": "activitytf", "geiger_counter": "geigercounters"}.get(lowered, lowered) + + def _match(jar: Jar, by_hash: dict[str, Release], prefixes: list[tuple[str, str]]) -> str | None: if jar.sha256 in by_hash: return by_hash[jar.sha256].repo lowered = jar.file.lower() - return next((repo for prefix, repo in prefixes if lowered.startswith(prefix + "-")), None) + matched = next((repo for prefix, repo in prefixes if lowered.startswith(prefix + "-")), None) + if matched: + return matched + # Stable Bukkit identifiers differ from repository names for these two plugins. + name = plugin_identifier(jar.name) + return next((repo for _, repo in prefixes if repo.lower() == name), None) def compare(releases: dict[str, list[Release]], servers: dict[str, dict]) -> tuple[list[dict], list[dict]]: @@ -496,22 +606,29 @@ def compare(releases: dict[str, list[Release]], servers: dict[str, dict]) -> tup for key, server in servers.items(): for jar in server["jars"]: repo = _match(jar, by_hash, prefixes) + if repo is None: + name = plugin_identifier(jar.name) + repo = next((r for r in getattr(releases, "errors", {}) if r.lower() == name), None) if repo is not None: grouped[key].setdefault(repo, []).append(jar) continue name = jar.name or jar.file.removesuffix(".jar") entry = others.setdefault(name.lower(), {"name": name, "servers": {k: None for k in servers}}) entry["servers"][key] = {"file": jar.file, "version": jar.version} + if getattr(releases, "error", None) or getattr(releases, "errors", {}): + entry["unverified"] = True plugins = [] - for repo in sorted(releases, key=str.lower): - latest = releases[repo][0] + for repo in sorted(set(releases) | set(getattr(releases, "errors", {})), key=str.lower): + latest = releases[repo][0] if repo in releases else None plugins.append({ "repo": repo, "latest": {"version": latest.version, "published_at": latest.published_at, "url": latest.url, - "branch": latest.branch}, - "servers": {key: _plugin_state(grouped[key].get(repo, []), latest, by_hash, - server["startup"], server["releases_only"]) + "branch": latest.branch} if latest else None, + **({"error": "github_unavailable"} if latest is None else {}), + "servers": {key: _plugin_state(grouped[key].get(repo, []), latest or Release(repo, "", "", "", "", ""), by_hash, + server["startup"], server["releases_only"], + repo in releases.complete if isinstance(releases, Releases) else True) for key, server in servers.items()}, }) other_list = [] @@ -527,27 +644,21 @@ def _now() -> str: def overview(fetch: Fetch | None = None) -> dict: - """The staff Servers tab: each server's state and every plugin it has.""" - server_info: list[dict] = [] - readable: dict[str, dict] = {} - for key, label, variable, releases_only in SERVERS: - root = os.getenv(variable, "").strip() - info: dict[str, Any] = {"key": key, "label": label, "releases_only": releases_only, - "status": "not_configured", "paper": None, "started": None, "log": None} - if root: - plugins = Path(root) / "plugins" - if plugins.is_dir(): - startup = read_startup(Path(root) / "logs") - info.update(status="available", paper=startup and startup["paper"], - started=startup and startup["done"], log=startup and startup["log"]) - readable[key] = {"jars": read_jars(plugins), "startup": startup, "releases_only": releases_only} - else: - info["status"] = "missing" - server_info.append(info) - - releases, checked_at, failed = releases_cache.get(fetch) - github = {"status": "available" if releases is not None else "unavailable", - "checked_at": checked_at, "stale": failed and releases is not None} - plugins, others = compare(releases or {}, readable) if releases is not None else ([], []) - return {"checked_at": _now(), "github": github, "servers": server_info, "plugins": plugins, "others": others} - + roots = {} + server_info = [] + for key, label, variable, only in SERVERS: + value = os.getenv(variable, "").strip() + available = bool(value) and (Path(value) / "plugins").is_dir() + server_info.append({"key": key, "label": label, "releases_only": only, + "status": "available" if available else "missing" if value else "not_configured", + "paper": None, "started": None, "log": None}) + if available: roots[key] = (Path(value), only) + releases, servers = check_servers(roots, fetch) + for info in server_info: + startup = servers.get(info["key"], {}).get("startup") + if startup: + info.update(paper=startup["paper"], started=startup["done"], log=startup["log"]) + plugins, others = compare(releases, servers) + status = "busy" if releases.error == "busy" else "unavailable" if releases.error else "partial" if releases.errors else "available" + return {"checked_at": _now(), "github": {"status": status, "checked_at": releases.checked_at}, + "servers": server_info, "plugins": plugins, "others": others} diff --git a/backend/src/servers/test_control.py b/backend/src/servers/test_control.py index 0c33b32d..7ac451be 100644 --- a/backend/src/servers/test_control.py +++ b/backend/src/servers/test_control.py @@ -706,3 +706,136 @@ def dispatch(self, body): assert refused.value.status == 409 server.shutdown() worker.join() + + +def test_latest_plan_fetches_fresh_releases_and_passes_verified_full_inventory(controller, monkeypatch): + from src.servers.test_inventory import Github, release + plugins, _ = controller.roots["dev"] + monkeypatch.setenv("SERVER_DEV_DIR", str(plugins.parent)) + monkeypatch.delenv("SERVERS_GITHUB_TOKEN", raising=False) + monkeypatch.delenv("PATCHNOTES_GITHUB_TOKEN", raising=False) + monkeypatch.delenv("SERVERS_GITHUB_ORG", raising=False) + baseline = controller.snapshot("dev") + rows = [release("v2.0.0", "2026-10-01T00:00:00Z", "core-2.0.0.jar", "2" * 64), + release("v1.0.0", "2026-09-01T00:00:00Z", "core-1.0.0.jar", baseline[0]["sha256"])] + fetch = Github({"Core": rows}) + monkeypatch.setattr(inventory, "_github", lambda path, body=None, **_: fetch(path, body)) + sent = [] + monkeypatch.setattr(control, "request", lambda operation, **body: sent.append((operation, body)) or {"id": "plan"}) + assert control.plan("dev", "latest") == {"id": "plan"} + rows.insert(0, release("v3.0.0", "2026-10-02T00:00:00Z", "core-3.0.0.jar", "3" * 64)) + control.plan("dev", "latest") + assert [body["releases"][0]["tag"] for _, body in sent] == ["v2.0.0", "v3.0.0"] + assert all(operation == "plan" and body["expected"] == baseline for operation, body in sent) + assert all({"sha256": baseline[0]["sha256"], "version": "1.0.0"} in body["releases"][0]["known"] for _, body in sent) + assert sum(path.startswith("/orgs/") for path, _ in fetch.calls) == 2 + assert controller.commands == [] + + +@pytest.mark.parametrize("failure", ["global", "partial", "history", "renamed"]) +def test_latest_plan_fails_closed_before_helper_for_unverified_releases(controller, monkeypatch, failure): + plugins, _ = controller.roots["dev"] + monkeypatch.setenv("SERVER_DEV_DIR", str(plugins.parent)) + monkeypatch.delenv("SERVERS_GITHUB_ORG", raising=False) + found = inventory.Releases() + found.repos.add("Core") + found["Core"] = [inventory.Release("Core", "2.0.0", "2026-10-01T00:00:00Z", "url", "core-2.0.0.jar", "2" * 64, "v2.0.0")] + if failure == "global": + found.error = "github_unavailable" + elif failure == "partial": + found.errors["Core"] = "github_unavailable" + elif failure == "renamed": + found.clear() + found.repos = {"DifferentRepo"} + local = {"dev": {"jars": inventory.read_jars(plugins), "startup": None, "releases_only": False}} + if failure == "renamed": + monkeypatch.setattr(inventory, "compare", lambda *_: ([], [{"name": "Core", "unverified": True}])) + monkeypatch.setattr(inventory, "check_servers", lambda *_: (found, local)) + monkeypatch.setattr(control, "request", lambda *_args, **_kwargs: pytest.fail("unverified data reached helper")) + with pytest.raises(control.ControlError, match="releases_unavailable"): + control.plan("dev", "latest") + assert controller.commands == [] + + +def test_latest_plan_refuses_an_old_helper_protocol_instead_of_retrying_unsafely(controller, monkeypatch): + plugins, _ = controller.roots["dev"] + monkeypatch.setenv("SERVER_DEV_DIR", str(plugins.parent)) + monkeypatch.delenv("SERVERS_GITHUB_ORG", raising=False) + found = inventory.Releases() + calls = [] + monkeypatch.setattr(inventory, "check_servers", lambda *_: (found, {"dev": {"jars": []}})) + def refused(operation, **body): + calls.append(body) + assert "expected" in body + raise control.ControlError("bad_control_request", 400) + monkeypatch.setattr(control, "request", refused) + with pytest.raises(control.ControlError, match="control_unavailable") as error: + control.plan("dev", "latest") + assert error.value.status == 503 and len(calls) == 1 + + +def test_restart_plan_keeps_legacy_protocol_without_a_github_check(monkeypatch): + monkeypatch.setattr(inventory, "check_servers", lambda *_: pytest.fail("restart looked up releases")) + sent = [] + monkeypatch.setattr(control, "request", lambda op, **body: sent.append((op, body)) or {"id": "legacy"}) + assert control.plan("dev", "restart") == {"id": "legacy"} + assert sent == [("plan", {"server": "dev", "mode": "restart", "releases": []})] + + +def test_helper_plan_accepts_old_protocol_and_matching_new_inventory(controller): + old = controller.dispatch({"operation": "plan", "server": "dev", "mode": "restart", "releases": []}) + expected = controller.snapshot("dev") + new = controller.dispatch({"operation": "plan", "server": "dev", "mode": "restart", "releases": [], "expected": expected}) + assert old["id"] != new["id"] + assert controller._plan(new["id"])["baseline"] == expected + + +def test_helper_rejects_changed_expected_inventory_before_commands_or_new_files(controller): + expected = controller.snapshot("dev") + old = controller.roots["dev"][0] / expected[0]["file"] + import os + stamp = old.stat() + raw = old.read_bytes() + old.write_bytes(raw[:-1] + bytes([raw[-1] ^ 1])) + os.utime(old, ns=(stamp.st_atime_ns, stamp.st_mtime_ns)) + before = set(controller.state.iterdir()) + with pytest.raises(control.ControlError, match="plugins_changed"): + controller.dispatch({"operation": "plan", "server": "dev", "mode": "latest", "releases": [], "expected": expected}) + assert controller.commands == [] + assert set(controller.state.iterdir()) == before + + +@pytest.mark.parametrize("expected", [None, "invalid", {}, [{}] * 301]) +def test_helper_rejects_malformed_new_inventory_field(controller, expected): + with pytest.raises(control.ControlError, match="bad_control_request"): + controller.dispatch({"operation": "plan", "server": "dev", "mode": "latest", "releases": [], "expected": expected}) + assert controller.commands == [] + + +def test_snapshot_hashes_each_jar_once(controller, monkeypatch): + original = inventory._describe + calls = [] + def describe(path): + calls.append(path) + return original(path) + monkeypatch.setattr(inventory, "_describe", describe) + monkeypatch.setattr(host, "digest", lambda *_: pytest.fail("snapshot hashed the jar twice")) + [copy] = controller.snapshot("dev") + assert calls == [controller.roots["dev"][0] / "core-1.0.0.jar"] + assert copy["name"] == "Core" and len(copy["sha256"]) == 64 + + +def test_latest_plan_upgrades_renamed_geiger_counter_from_its_verified_old_hash(controller, tmp_path): + plugins, _ = controller.roots["dev"] + old_sha = jar(plugins / "radiation-old.jar", name="geiger_counter", version="1.0.0") + target = tmp_path / "geiger-release.jar" + new_sha = jar(target, name="geiger_counter", version="2.0.0") + controller.fetch = lambda release, path: path.write_bytes(target.read_bytes()) + release = {"repo": "GeigerCounters", "tag": "v2.0.0", "asset": "geigercounters-2.0.0.jar", "sha256": new_sha, + "known": [{"sha256": old_sha, "version": "1.0.0"}]} + plan = controller.plan("dev", "latest", [release], expected=controller.snapshot("dev")) + [change] = plan["changes"] + assert change["repo"] == "GeigerCounters" and change["name"] == "geiger_counter" + assert change["old_file"] == "radiation-old.jar" and change["old_version"] == "1.0.0" + assert change["sha256"] == new_sha and "test_build" not in change + assert plan["skipped"] == [] diff --git a/backend/src/servers/test_inventory.py b/backend/src/servers/test_inventory.py index f572d3f6..1ff875ed 100644 --- a/backend/src/servers/test_inventory.py +++ b/backend/src/servers/test_inventory.py @@ -1,8 +1,10 @@ import gzip import hashlib import http.client +import os import threading import urllib.error +import urllib.parse import zipfile import pytest @@ -14,8 +16,9 @@ def make_jar(folder, file, name, version, extra=b""): folder.mkdir(parents=True, exist_ok=True) path = folder / file with zipfile.ZipFile(path, "w") as archive: - archive.writestr("plugin.yml", f"name: {name}\nversion: '{version}'\nmain: x.Y\n") - archive.writestr("x/Y.class", extra or file.encode()) + for entry, content in (("plugin.yml", f"name: {name}\nversion: '{version}'\nmain: x.Y\n"), + ("x/Y.class", extra or file.encode())): + archive.writestr(zipfile.ZipInfo(entry, date_time=(2026, 10, 15, 0, 0, 0)), content) return hashlib.sha256(path.read_bytes()).hexdigest() @@ -35,47 +38,15 @@ def startup_log(*plugins, paper="1.21.10-130", done=True): @pytest.fixture(autouse=True) -def fresh_cache(monkeypatch): - monkeypatch.delenv("SERVERS_GITHUB_TOKEN", raising=False) - monkeypatch.delenv("PATCHNOTES_GITHUB_TOKEN", raising=False) - inventory.releases_cache.clear() - yield - inventory.releases_cache.clear() - - -def test_releases_skip_drafts_prereleases_and_releases_without_a_jar(): - data = [ - release("v1.1.0", None, "core-1.1.0.jar", "d1", draft=True), - release("v1.0.1", "2026-10-02T00:00:00Z", "core-1.0.1.jar", "b2", prerelease=True), - release("v1.0.0", "2026-10-01T00:00:00Z", "core-1.0.0.jar", "A1"), - release("v1.0.2", "2026-10-03T00:00:00Z", "core-1.0.2.tar.gz", "c3"), - release("2.0.0", "2026-10-04T00:00:00Z", "core-2.0.0.jar", "e4"), - ] - found = inventory.repo_releases("Core", data) - assert [(r.version, r.sha256, r.prefix) for r in found] == [("2.0.0", "e4", "core"), ("1.0.0", "a1", "core")] - assert inventory.repo_releases("Core", {"message": "Not Found"}) == [] - odd = release("v3.0.0", "2026-10-05T00:00:00Z", "core-3.0.0.jar", "f5") - odd["assets"].insert(0, "not an asset") - assert [r.version for r in inventory.repo_releases("Core", [odd])] == ["3.0.0"] - - -def test_startup_comes_from_the_newest_log_that_has_one(tmp_path): - logs = tmp_path / "logs" - logs.mkdir() - # After midnight, latest.log holds no startup; the rotated logs are read newest first. - (logs / "latest.log").write_text("[00:01:00] [Server thread/INFO]: nothing\n") - with gzip.open(logs / "2026-10-10-2.log.gz", "wt") as handle: - handle.write(startup_log(("Core", "0.9"), paper="old")) - with gzip.open(logs / "2026-10-10-10.log.gz", "wt") as handle: - handle.write(startup_log(("Core", "1.1"), done=False)) - startup = inventory.read_startup(logs) - assert startup == {"paper": "1.21.10-130", "done": False, "log": "2026-10-10-10.log.gz", - "enabled": {"core": "1.1"}, "failed": {}} - assert inventory.read_startup(tmp_path / "nowhere") is None +def fresh_environment(monkeypatch): + for name in ("SERVERS_GITHUB_TOKEN", "PATCHNOTES_GITHUB_TOKEN", "SERVERS_GITHUB_ORG", + "SERVER_MAIN_DIR", "SERVER_DEV_DIR"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(inventory, "_check_slots", threading.BoundedSemaphore(2)) def servers_fixture(tmp_path): - """Main and Dev folders with a release, an older release, a test build and a pending update.""" + """Main and Dev include an old release, test build, duplicate and pending update.""" main, dev = tmp_path / "main", tmp_path / "dev" shas = { "core-new": make_jar(tmp_path / "rel", "core-1.1.0.jar", "Core", "1.1.0"), @@ -105,88 +76,143 @@ def repo_data(shas): } -def github(shas, fail=False): - """GitHub's REST API, as read without a token.""" - calls = [] - - def fetch(path, body=None): - calls.append(path) - if fail: - raise urllib.error.URLError("down") - assert body is None, "GraphQL needs a token" - repos = repo_data(shas) +class Github: + """A strict REST/GraphQL fixture with independent discovery, latest and history calls.""" + def __init__(self, repos, *, failed=(), history_failed=(), heads=None, commits=None, latest=None): + self.repos = repos + self.failed = set(failed) + self.history_failed = set(history_failed) + self.heads = heads or {} + self.commits = commits or {} + self.latest = latest or {} + self.calls = [] + + @property + def queries(self): + return [body for path, body in self.calls if path == "/graphql"] + + def node(self, repo, row): + assets = row["assets"] + return {"tagName": row["tag_name"], "publishedAt": row["published_at"], "isDraft": row["draft"], + "isPrerelease": row["prerelease"], "url": row["html_url"], + "tagCommit": {"oid": self.commits.get(repo, "same")}, + "releaseAssets": {"nodes": assets[:20], "pageInfo": { + "hasNextPage": len(assets) > 20, "endCursor": "20"}}} + + def latest_row(self, repo): + rows = self.repos[repo] + return next((r for r in rows if r["tag_name"] == self.latest[repo]), None) if repo in self.latest else next(iter(rows), None) + + def __call__(self, path, body=None): + self.calls.append((path, body)) if path.startswith("/orgs/"): - return [{"name": name} for name in repos] + [{"name": "Old", "archived": True}] - if "/Old/" in path: - raise AssertionError("archived repositories are not read") - return next((found for name, found in repos.items() if f"/{name}/" in path), []) - - fetch.calls = calls - return fetch - + assert body is None + return [{"name": name} for name in self.repos] + [{"name": "Old", "archived": True}] + if path != "/graphql": + assert body is None + parts = urllib.parse.urlsplit(path) + repo = parts.path.split("/")[3] + assert repo != "Old", "archived repositories must be excluded" + if repo in self.failed: + raise urllib.error.URLError("repo down") + if parts.path.endswith("/releases/latest"): + row = self.latest_row(repo) + if row is None: + raise urllib.error.HTTPError(path, 404, "not found", {}, None) + return row + assert parts.path.endswith("/releases") + assert repo not in self.history_failed or self._history_failure() + query = urllib.parse.parse_qs(parts.query) + assert query["per_page"] == ["30"] + start = (int(query["page"][0]) - 1) * 30 + return self.repos[repo][start:start + 30] + query, variables = body["query"], body["variables"] + if "organization(login" in query: + assert "defaultBranchRef" not in query + names = list(self.repos) + page = names[:2] if variables["after"] is None else names[2:] + return {"data": {"organization": {"repositories": {"nodes": [{"name": n} for n in page], + "pageInfo": {"hasNextPage": variables["after"] is None and len(names) > 2, "endCursor": "repos2"}}}}} + if "latestRelease" in query: + data = {} + for alias in sorted(k for k in variables if k.startswith("r") and k[1:].isdigit()): + repo = variables[alias] + row = self.latest_row(repo) + data[alias] = None if repo in self.failed else { + "name": repo, "defaultBranchRef": {"name": "master" if repo == "Lib" else "main", + "target": {"oid": self.heads.get(repo, "same")}}, + "latestRelease": self.node(repo, row) if row else None} + return {"data": data, **({"errors": [{"type": "NOT_FOUND"}]} if self.failed else {})} + if "compare(headRef" in query: + return {"data": {alias: {"ref": {"compare": {"status": "AHEAD", "aheadBy": 3}}} + for alias in variables if alias.startswith("r") and alias[1:].isdigit()}} + repo = variables["repo"] + if "releaseAssets(first:20,after:" in query: + row = next(r for r in self.repos[repo] if r["tag_name"] == variables["tag"]) + start = int(variables["after"]) + assets = row["assets"][start:start + 20] + return {"data": {"repository": {"release": {"releaseAssets": {"nodes": assets, + "pageInfo": {"hasNextPage": start + 20 < len(row["assets"]), "endCursor": str(start + 20)}}}}}} + if "releases(first:30" in query: + if repo in self.history_failed: + self._history_failure() + start = int(variables["after"] or 0) + rows = self.repos[repo][start:start + 30] + return {"data": {"repository": {"releases": {"nodes": [self.node(repo, r) for r in rows], + "pageInfo": {"hasNextPage": start + 30 < len(self.repos[repo]), "endCursor": str(start + 30)}}}}} + assert "release(tagName:$t" in query, query + tags = {k: tag for k, tag in variables.items() if k.startswith("t") and k[1:].isdigit()} + return {"data": {"repository": {alias: next((self.node(repo, row) for row in self.repos[repo] + if row["tag_name"] == tag), None) for alias, tag in tags.items()}}} + + @staticmethod + def _history_failure(): + raise http.client.IncompleteRead(b"") -def graphql(shas): - """GitHub's GraphQL API, as read with a token.""" - queries = [] - repos = repo_data(shas) - branches = {"Core": "main", "Lib": "master", "Docs": "main"} - def node(r): - return {"tagName": r["tag_name"], "publishedAt": r["published_at"], "isDraft": r["draft"], - "isPrerelease": r["prerelease"], "url": r["html_url"], "releaseAssets": {"nodes": r["assets"]}} +def github(shas, fail=False): + if fail: + def failed(path, body=None, **_kwargs): + raise urllib.error.URLError("down") + return failed + return Github(repo_data(shas)) - def fetch(path, body=None): - assert path == "/graphql" - query, variables = body["query"], body["variables"] - queries.append(body) - aliases = sorted(k for k in variables if k[0] == "r") - if "pageInfo" in query: - # Two pages, to follow the cursor. - names = ["Core", "Lib"] if variables["after"] is None else ["Docs"] - nodes = [{"name": n, "defaultBranchRef": {"name": branches[n]}} for n in names] - return {"data": {"organization": {"repositories": { - "pageInfo": {"hasNextPage": variables["after"] is None, "endCursor": "c1"}, "nodes": nodes}}}} - if "compare" in query: - data = {} - for alias in aliases: - # GitHub can't find Lib's tag. - data[alias] = {"ref": None} if variables[alias] == "Lib" else {"ref": {"compare": { - "status": "AHEAD", "aheadBy": 3}}} - return {"data": data} - return {"data": {a: {"releases": {"nodes": [node(r) for r in repos[variables[a]]]}} for a in aliases}} - fetch.queries = queries - return fetch +def test_releases_skip_drafts_prereleases_and_releases_without_a_jar(): + data = [release("v1.1.0", None, "core-1.1.0.jar", "d1", draft=True), + release("v1.0.1", "2026-10-02T00:00:00Z", "core-1.0.1.jar", "b2", prerelease=True), + release("v1.0.0", "2026-10-01T00:00:00Z", "core-1.0.0.jar", "A1"), + release("v1.0.2", "2026-10-03T00:00:00Z", "core-1.0.2.tar.gz", "c3"), + release("2.0.0", "2026-10-04T00:00:00Z", "core-2.0.0.jar", "e4")] + found = inventory.repo_releases("Core", data) + assert [(r.version, r.sha256, r.prefix) for r in found] == [("2.0.0", "e4", "core"), ("1.0.0", "a1", "core")] + assert inventory.repo_releases("Core", {"message": "Not Found"}) == [] + odd = release("v3.0.0", "2026-10-05T00:00:00Z", "core-3.0.0.jar", "f5") + odd["assets"].insert(0, "not an asset") + assert [r.version for r in inventory.repo_releases("Core", [odd])] == ["3.0.0"] -def test_overview_compares_each_server_with_the_releases(tmp_path, monkeypatch): +def test_overview_compares_each_server_with_verified_releases(tmp_path, monkeypatch): main, dev, shas = servers_fixture(tmp_path) monkeypatch.setenv("SERVER_MAIN_DIR", str(main)) monkeypatch.setenv("SERVER_DEV_DIR", str(dev)) result = inventory.overview(github(shas)) - - assert result["github"]["status"] == "available" and not result["github"]["stale"] + assert result["github"]["status"] == "available" and result["github"]["checked_at"] + assert "stale" not in result["github"] assert [s["status"] for s in result["servers"]] == ["available", "available"] assert result["servers"][0]["paper"] == "1.21.10-130" and result["servers"][0]["started"] is True plugins = {p["repo"]: p for p in result["plugins"]} assert set(plugins) == {"Core", "Lib"} assert plugins["Core"]["latest"]["version"] == "1.1.0" - # Without a token, releases aren't compared with the branch. assert plugins["Core"]["latest"]["branch"] is None - core_main = plugins["Core"]["servers"]["main"] assert core_main["state"] == "older_release" and core_main["release"] == "1.0.0" assert core_main["update"] is True and core_main["loaded_state"] == "same" assert core_main["files"] == ["core-1.0.0.jar", "core-1.1.0.jar"] - - # Same name and version as the release, different bytes: Main only runs releases. - lib_main = plugins["Lib"]["servers"]["main"] - assert lib_main["state"] == "not_release" and lib_main["release"] is None - + assert plugins["Lib"]["servers"]["main"]["state"] == "not_release" core_dev = plugins["Core"]["servers"]["dev"] assert core_dev["state"] == "test_build" and core_dev["duplicate"] is True assert core_dev["loaded"] == "main-SNAPSHOT" and core_dev["loaded_state"] == "same" - assert plugins["Lib"]["servers"]["dev"]["state"] == "missing" assert result["others"] == [{"name": "Essentials", "differs": True, "servers": { "main": {"file": "Essentials-2.22.jar", "version": "2.22"}, @@ -197,11 +223,9 @@ def test_a_newer_jar_than_the_running_one_waits_for_a_restart(tmp_path, monkeypa main, _, shas = servers_fixture(tmp_path) (main / "logs" / "latest.log").write_text(startup_log(("Core", "0.9.0"))) monkeypatch.setenv("SERVER_MAIN_DIR", str(main)) - monkeypatch.delenv("SERVER_DEV_DIR", raising=False) result = inventory.overview(github(shas)) plugins = {p["repo"]: p["servers"]["main"] for p in result["plugins"]} assert plugins["Core"]["loaded"] == "0.9.0" and plugins["Core"]["loaded_state"] == "different" - # Lib never enabled in a startup that finished. assert plugins["Lib"]["loaded_state"] == "not_loaded" assert result["servers"][1]["status"] == "not_configured" assert set(result["plugins"][0]["servers"]) == {"main"} @@ -209,179 +233,231 @@ def test_a_newer_jar_than_the_running_one_waits_for_a_restart(tmp_path, monkeypa def test_unreadable_servers_and_github(tmp_path, monkeypatch): monkeypatch.setenv("SERVER_MAIN_DIR", str(tmp_path / "gone")) - monkeypatch.setenv("SERVER_DEV_DIR", "") result = inventory.overview(github({}, fail=True)) assert [s["status"] for s in result["servers"]] == ["missing", "not_configured"] - assert result["github"] == {"status": "unavailable", "checked_at": None, "stale": False} + assert result["github"] == {"status": "unavailable", "checked_at": None} assert result["plugins"] == [] and result["others"] == [] -def test_releases_are_cached_and_kept_when_github_fails(tmp_path, monkeypatch): - main, _, shas = servers_fixture(tmp_path) - monkeypatch.setenv("SERVER_MAIN_DIR", str(main)) +@pytest.mark.parametrize("authenticated", [False, True]) +def test_every_check_reads_the_designated_latest_release_fresh(tmp_path, monkeypatch, authenticated): + _, _, shas = servers_fixture(tmp_path) + if authenticated: + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") fetch = github(shas) - inventory.overview(fetch) - calls = len(fetch.calls) - inventory.overview(fetch) - assert len(fetch.calls) == calls - - inventory.releases_cache._next = 0 - inventory.overview(github(shas, fail=True)) - inventory.releases_cache.wait_for_refresh() - result = inventory.overview(github(shas, fail=True)) - assert result["github"]["status"] == "available" and result["github"]["stale"] is True - assert {p["repo"] for p in result["plugins"]} == {"Core", "Lib"} - - -def test_with_a_token_releases_come_from_graphql_and_are_compared_with_the_default_branch(tmp_path, monkeypatch): - main, _, shas = servers_fixture(tmp_path) - monkeypatch.setenv("SERVER_MAIN_DIR", str(main)) - monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") - fetch = graphql(shas) - result = inventory.overview(fetch) - plugins = {p["repo"]: p for p in result["plugins"]} - assert set(plugins) == {"Core", "Lib"} - assert plugins["Core"]["servers"]["main"]["state"] == "older_release" - assert plugins["Core"]["latest"]["branch"] == {"name": "main", "status": "ahead", "ahead": 3, - "url": "https://github.com/TF-Minecraft/Core/compare/v1.1.0...main"} - # GitHub couldn't compare Lib's release with master, so nothing is claimed. - assert plugins["Lib"]["latest"]["branch"] is None - # Two pages of repositories, one releases query and one comparison of the latest releases. - assert len(fetch.queries) == 4 - compare = fetch.queries[-1]["variables"] - assert {compare["t0"], compare["b0"], compare["t1"], compare["b1"]} == { - "refs/tags/v1.1.0", "main", "refs/tags/v2.0.0", "master"} - - -def test_queries_are_split_into_small_groups(tmp_path, monkeypatch): + first = inventory.fetch_releases(fetch) + fetch.repos["Core"].insert(0, release("v1.2.0", "2026-10-03T00:00:00Z", "core-1.2.0.jar", "new")) + second = inventory.fetch_releases(fetch) + assert first["Core"][0].version == "1.1.0" + assert second["Core"][0].version == "1.2.0" + assert first is not second + assert first.error is None and second.checked_at + if authenticated: + assert sum("organization(login" in b["query"] for b in fetch.queries) == 4 + else: + assert sum(p.startswith("/orgs/") for p, _ in fetch.calls) == 2 + + +@pytest.mark.parametrize("authenticated", [False, True]) +def test_latest_is_github_designation_instead_of_newest_published_date(monkeypatch, authenticated): + if authenticated: + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + fetch = Github({"Core": [release("v1.0.1", "2026-10-15T00:00:00Z", "core-1.0.1.jar", "old"), + release("v2.0.0", "2026-10-01T00:00:00Z", "core-2.0.0.jar", "new")]}, + latest={"Core": "v2.0.0"}) + assert inventory.fetch_releases(fetch)["Core"][0].version == "2.0.0" + + +def test_identical_commits_annotated_tag_commit_skip_compare(tmp_path, monkeypatch): _, _, shas = servers_fixture(tmp_path) monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") - monkeypatch.setattr(inventory, "GRAPHQL_CHUNK", 1) - fetch = graphql(shas) - assert set(inventory.fetch_releases(fetch)) == {"Core", "Lib"} - # 2 repository pages, then releases for each of 3 repositories and comparisons for the 2 with releases. - assert len(fetch.queries) == 7 - - -def test_a_bad_graphql_answer_fails_the_check(monkeypatch): - monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") - releases, _, failed = inventory.releases_cache.get(lambda path, body=None: {"errors": [{"message": "nope"}]}) - assert releases is None and failed is True + fetch = github(shas) + found = inventory.fetch_releases(fetch) + assert found["Core"][0].branch["status"] == "identical" + assert found["Lib"][0].branch["name"] == "master" + assert not any("compare(headRef" in q["query"] for q in fetch.queries) + latest_queries = [q["query"] for q in fetch.queries if "latestRelease" in q["query"]] + assert latest_queries and all("tagCommit { oid }" in q for q in latest_queries) + assert all("target { oid }" in q for q in latest_queries) -def test_after_the_first_check_releases_refresh_in_the_background(tmp_path, monkeypatch): +def test_unequal_commits_only_compare_mismatches_and_unknown_tag_commit_stays_unknown(tmp_path, monkeypatch): _, _, shas = servers_fixture(tmp_path) - first, _, _ = inventory.releases_cache.get(github(shas)) - inventory.releases_cache._next = 0 - slow = threading.Event() - - def waiting(path, body=None): - slow.wait(5) - raise urllib.error.URLError("down") - - # The expired releases come back at once while GitHub is asked again. - again, _, failed = inventory.releases_cache.get(waiting) - assert again is first and failed is False - slow.set() - inventory.releases_cache.wait_for_refresh() - kept, _, failed = inventory.releases_cache.get(waiting) - assert kept is first and failed is True - + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + fetch = Github(repo_data(shas), heads={"Core": "head"}, commits={"Lib": None}) + found = inventory.fetch_releases(fetch) + assert found["Core"][0].branch == {"name": "main", "status": "ahead", "ahead": 3, + "url": "https://github.com/TF-Minecraft/Core/compare/v1.1.0...main"} + assert found["Lib"][0].branch is None + [compared] = [q for q in fetch.queries if "compare(headRef" in q["query"]] + assert compared["variables"] == {"org": "TF-Minecraft", "r0": "Core", "t0": "refs/tags/v1.1.0", "b0": "main"} -def test_github_is_read_by_one_fetch_at_a_time(tmp_path): - _, _, shas = servers_fixture(tmp_path) - inventory.releases_cache.get(github(shas)) - inventory.releases_cache._next = 0 - entered, release_it = threading.Event(), threading.Event() - fetches = [] - def slow(path, body=None): - if path.startswith("/orgs/"): - fetches.append(path) - entered.set() - release_it.wait(5) - return github(shas)(path) - - def unused(path, body=None): - fetches.append("second") - return github(shas)(path) - - inventory.releases_cache.get(slow) # starts the background refresh - assert entered.wait(5) - # Another page load and a waiting caller share the refresh in flight. - inventory.releases_cache.get(unused) - waited = [] - waiter = threading.Thread(target=lambda: waited.append(inventory.releases_cache.get(unused, wait=True))) - waiter.start() - waiter.join(0.1) - assert waiter.is_alive() - release_it.set() - waiter.join(5) - [(releases, _, failed)] = waited - assert releases is not None and failed is False and len(fetches) == 1 - - -def test_any_fetch_error_is_recorded_for_a_waiting_caller(tmp_path): +def test_latest_queries_are_split_into_bounded_chunks(tmp_path, monkeypatch): _, _, shas = servers_fixture(tmp_path) - inventory.releases_cache.get(github(shas)) - inventory.releases_cache._next = 0 - - def cut_short(path, body=None): - raise http.client.IncompleteRead(b"") - - releases, _, failed = inventory.releases_cache.get(cut_short, wait=True) - assert releases is not None and failed is True + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + monkeypatch.setattr(inventory, "GRAPHQL_CHUNK", 1) + fetch = github(shas) + found = inventory.fetch_releases(fetch) + assert set(found) == {"Core", "Lib"} and found.repos == {"Core", "Lib", "Docs"} + assert len([q for q in fetch.queries if "latestRelease" in q["query"]]) == 3 -def test_a_fetch_running_when_the_cache_is_cleared_is_ignored(tmp_path): - _, _, shas = servers_fixture(tmp_path) - entered, release_it = threading.Event(), threading.Event() +def test_partial_latest_failure_keeps_successful_repo_and_local_plugin_visible(tmp_path, monkeypatch): + main, _, shas = servers_fixture(tmp_path) + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + monkeypatch.setenv("SERVER_MAIN_DIR", str(main)) + result = inventory.overview(Github(repo_data(shas), failed={"Core"})) + assert result["github"]["status"] == "partial" + plugins = {p["repo"]: p for p in result["plugins"]} + assert plugins["Lib"]["latest"]["version"] == "2.0.0" + assert plugins["Core"]["latest"] is None and plugins["Core"]["error"] == "github_unavailable" + assert plugins["Core"]["servers"]["main"]["state"] == "unverified" + assert plugins["Core"]["servers"]["main"]["version"] == "1.0.0" - def slow(path, body=None): - entered.set() - release_it.wait(5) - return github(shas)(path) - waiter = threading.Thread(target=lambda: inventory.releases_cache.get(slow)) - waiter.start() - assert entered.wait(5) - inventory.releases_cache.clear() - release_it.set() - waiter.join(5) - assert inventory.releases_cache._state() == (None, None, False) +def test_global_failure_does_not_reuse_previous_releases_and_keeps_local_inventory(tmp_path, monkeypatch): + main, _, shas = servers_fixture(tmp_path) + monkeypatch.setenv("SERVER_MAIN_DIR", str(main)) + inventory.overview(github(shas)) + failed = inventory.overview(github({}, fail=True)) + assert failed["github"] == {"status": "unavailable", "checked_at": None} + assert failed["plugins"] == [] + assert {p["name"] for p in failed["others"]} == {"Core", "Lib", "Essentials"} + assert all(p["unverified"] for p in failed["others"]) + assert failed["servers"][0]["paper"] == "1.21.10-130" -def test_a_graphql_error_in_releases_fails_the_check_but_not_in_comparisons(tmp_path, monkeypatch): - _, _, shas = servers_fixture(tmp_path) +def test_bad_graphql_answer_fails_fresh_check(monkeypatch): monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") - fetch = graphql(shas) - - def missing_repository(path, body=None): - answer = fetch(path, body) - if "...releases" in body["query"]: - answer["data"]["r0"] = None - answer["errors"] = [{"type": "NOT_FOUND", "path": ["r0"]}] - return answer + found = inventory.fetch_releases(lambda path, body=None: {"errors": [{"message": "nope"}]}) + assert found.error == "github_unavailable" and not found and found.checked_at is None - releases, _, failed = inventory.releases_cache.get(missing_repository) - assert releases is None and failed is True - def compare_error(path, body=None): - answer = fetch(path, body) - if "compare" in body["query"]: - answer["errors"] = [{"type": "NOT_FOUND", "path": ["r1", "ref"]}] +def test_exact_old_release_tag_verifies_hash_without_history(tmp_path, monkeypatch): + main = tmp_path / "main" + sha = make_jar(main / "plugins", "renamed.jar", "Core", "1.0.0") + rows = [release("v2.0.0", "2026-10-02T00:00:00Z", "core-2.0.0.jar", "new"), + release("v1.0.0", "2026-10-01T00:00:00Z", "core-1.0.0.jar", sha)] + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + fetch = Github({"Core": rows}) + found, servers = inventory.check_servers({"main": (main, True)}, fetch) + [plugin], others = inventory.compare(found, servers) + assert plugin["servers"]["main"]["state"] == "older_release" and not others + assert plugin["latest"]["version"] == "2.0.0" + assert any("release(tagName:$t" in q["query"] for q in fetch.queries) + assert not any("releases(first:30" in q["query"] for q in fetch.queries) + assert {v for q in fetch.queries for k, v in q["variables"].items() if k.startswith("t")} >= {"v1.0.0", "1.0.0"} + + +@pytest.mark.parametrize("authenticated", [False, True]) +def test_history_paginates_past_thirty_and_recognises_renamed_old_jar(tmp_path, monkeypatch, authenticated): + if authenticated: + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + root = tmp_path / "main" + sha = make_jar(root / "plugins", "old-engine-custom.jar", "Core", "legacy") + rows = [release(f"v2.0.{i}", "2026-10-15T00:00:00Z", f"core-2.0.{i}.jar", str(i)) for i in range(31)] + rows.append(release("v0.9.0", "2025-01-01T00:00:00Z", "ancient-0.9.0.jar", sha)) + fetch = Github({"Core": rows}) + found, servers = inventory.check_servers({"main": (root, True)}, fetch) + [plugin], others = inventory.compare(found, servers) + assert plugin["servers"]["main"]["state"] == "older_release" + assert plugin["servers"]["main"]["release"] == "0.9.0" and not others + if authenticated: + pages = [q["variables"]["after"] for q in fetch.queries if "releases(first:30" in q["query"]] + assert pages == [None, "30"] + else: + assert any("per_page=30&page=2" in p for p, _ in fetch.calls) + + +def test_asset_connection_paginates_before_resolving_old_hash(tmp_path, monkeypatch): + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + root = tmp_path / "main" + sha = make_jar(root / "plugins", "renamed.jar", "Core", "1.0.0") + old = release("v1.0.0", "2026-10-01T00:00:00Z", "core-1.0.0.jar", sha) + old["assets"] = [{"name": f"notes-{n}.txt", "digest": "sha256:notes"} for n in range(25)] + old["assets"] + fetch = Github({"Core": [release("v2.0.0", "2026-10-02T00:00:00Z", "core-2.0.0.jar", "new"), old]}) + found, servers = inventory.check_servers({"main": (root, True)}, fetch) + [plugin], _ = inventory.compare(found, servers) + assert plugin["servers"]["main"]["state"] == "older_release" + assert any("releaseAssets(first:20,after:" in q["query"] for q in fetch.queries) + + +@pytest.mark.parametrize("authenticated", [False, True]) +def test_incomplete_history_stays_unverified_instead_of_claiming_a_test_build(tmp_path, monkeypatch, authenticated): + if authenticated: + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + root = tmp_path / "dev" + make_jar(root / "plugins", "core-branch.jar", "Core", "main-SNAPSHOT") + fetch = Github({"Core": [release("v2.0.0", "2026-10-02T00:00:00Z", "core-2.0.0.jar", "new")]}, history_failed={"Core"}) + found, servers = inventory.check_servers({"dev": (root, False)}, fetch) + [plugin], _ = inventory.compare(found, servers) + assert plugin["servers"]["dev"]["state"] == "unverified" + assert "Core" not in found.complete + + +def test_same_size_same_mtime_jar_bytes_are_read_again(tmp_path): + plugins = tmp_path / "plugins" + first = make_jar(plugins, "core.jar", "Core", "1.0.0", b"AAAA") + [original] = inventory.read_jars(plugins) + path = plugins / "core.jar" + stamp = path.stat() + second = make_jar(plugins, "core.jar", "Core", "1.0.0", b"BBBB") + os.utime(path, ns=(stamp.st_atime_ns, stamp.st_mtime_ns)) + [changed] = inventory.read_jars(plugins) + assert path.stat().st_size == stamp.st_size + assert path.stat().st_mtime_ns == stamp.st_mtime_ns + assert original.sha256 == first and changed.sha256 == second and first != second + + +def test_two_checks_have_independent_results_and_third_returns_busy_with_local_files(tmp_path, monkeypatch): + root = tmp_path / "dev" + make_jar(root / "plugins", "core.jar", "Core", "1.0.0") + entered = [threading.Event(), threading.Event()] + release_work = threading.Event() + results = [None, None] + calls = [] + workers = [] + def fetch(index): + def answer(path, body=None): + calls.append((index, path)) + if path.startswith("/orgs/"): + entered[index].set() + assert release_work.wait(5), "test did not release GitHub worker" + return Github({"Core": [release(f"v{index + 1}.0.0", "2026-10-01T00:00:00Z", f"core-{index + 1}.0.0.jar", str(index))]})(path, body) return answer + def run(index): + results[index] = inventory.fetch_releases(fetch(index)) + try: + for i in range(2): + worker = threading.Thread(target=run, args=(i,)) + workers.append(worker) + worker.start() + assert entered[i].wait(3) + busy, local = inventory.check_servers({"dev": (root, False)}, lambda *_: pytest.fail("busy check fetched GitHub")) + assert busy.error == "busy" and busy.checked_at is None + assert local["dev"]["jars"][0].name == "Core" + assert {i for i, path in calls if path.startswith("/orgs/")} == {0, 1} + finally: + release_work.set() + for worker in workers: + worker.join(5) + assert not worker.is_alive() + assert [r["Core"][0].version for r in results] == ["1.0.0", "2.0.0"] +def test_startup_comes_from_the_newest_log_that_has_one(tmp_path): + logs = tmp_path / "logs" + logs.mkdir() + # After midnight, latest.log holds no startup; the rotated logs are read newest first. + (logs / "latest.log").write_text("[00:01:00] [Server thread/INFO]: nothing\n") + with gzip.open(logs / "2026-10-10-2.log.gz", "wt") as handle: + handle.write(startup_log(("Core", "0.9"), paper="old")) + with gzip.open(logs / "2026-10-10-10.log.gz", "wt") as handle: + handle.write(startup_log(("Core", "1.1"), done=False)) + startup = inventory.read_startup(logs) + assert startup == {"paper": "1.21.10-130", "done": False, "log": "2026-10-10-10.log.gz", + "enabled": {"core": "1.1"}, "failed": {}} + assert inventory.read_startup(tmp_path / "nowhere") is None - releases = inventory.fetch_releases(compare_error) - assert releases["Core"][0].branch["status"] == "ahead" and releases["Lib"][0].branch is None - - -def test_a_caller_can_wait_for_fresh_releases(tmp_path, monkeypatch): - _, _, shas = servers_fixture(tmp_path) - inventory.releases_cache.get(github(shas)) - inventory.releases_cache._next = 0 - releases, _, failed = inventory.releases_cache.get(github(shas, fail=True), wait=True) - assert releases is not None and failed is True def test_unreadable_jar_keeps_its_hash_without_a_version(tmp_path): @@ -439,3 +515,166 @@ def test_plugin_text_quoting_an_enable_failure_is_not_a_failure(tmp_path): "[06:10:40] [Server thread/INFO]: Done (40.0s)! For help, type \"help\"\n") startup = inventory.read_startup(tmp_path) assert startup["enabled"] == {"core": "1.1"} and startup["failed"] == {} + + +def test_partial_graphql_error_cannot_disguise_failed_latest_as_no_releases(monkeypatch): + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + base = Github({"Core": [release("v1.0.0", "2026-10-01T00:00:00Z", "core-1.0.0.jar", "known")]}) + def failed_latest(path, body=None): + result = base(path, body) + if "latestRelease" in body["query"]: + result["data"]["r0"]["latestRelease"] = None + result["errors"] = [{"message": "temporary failure", "path": ["r0", "latestRelease"]}] + return result + found = inventory.fetch_releases(failed_latest) + assert found.errors == {"Core": "github_unavailable"} and not found + + +def test_failed_branch_compare_keeps_latest_release_without_claiming_identity(monkeypatch): + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + base = Github({"Core": [release("v1.0.0", "2026-10-01T00:00:00Z", "core-1.0.0.jar", "known")]}, + heads={"Core": "changed"}) + def failed_compare(path, body=None): + if "compare(headRef" in body["query"]: + return {"data": {"r0": {"ref": None}}, "errors": [{"path": ["r0", "ref"], "message": "unknown tag"}]} + return base(path, body) + found = inventory.fetch_releases(failed_compare) + assert found["Core"][0].version == "1.0.0" and found["Core"][0].branch is None + + +@pytest.mark.parametrize("authenticated", [False, True]) +def test_failure_after_full_first_history_page_cannot_prove_unreleased_bytes(tmp_path, monkeypatch, authenticated): + if authenticated: + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + root = tmp_path / "dev" + make_jar(root / "plugins", "core-branch.jar", "Core", "branch-SNAPSHOT") + base = Github({"Core": [release(f"v2.0.{i}", "2026-10-01T00:00:00Z", f"core-2.0.{i}.jar", str(i)) for i in range(31)]}) + def partial_history(path, body=None): + if (body and "releases(first:30" in body["query"] and body["variables"]["after"] == "30") or "page=2" in path: + raise urllib.error.URLError("history page two down") + return base(path, body) + found, local = inventory.check_servers({"dev": (root, False)}, partial_history) + [plugin], _ = inventory.compare(found, local) + assert plugin["servers"]["dev"]["state"] == "unverified" + assert "Core" not in found.complete + + +def test_latest_asset_connection_paginates_beyond_first_twenty_assets(monkeypatch): + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + row = release("v1.0.0", "2026-10-01T00:00:00Z", "core-1.0.0.jar", "known") + row["assets"] = [{"name": f"notes-{n}.txt", "digest": "sha256:notes"} for n in range(25)] + row["assets"] + fetch = Github({"Core": [row]}) + found = inventory.fetch_releases(fetch) + assert found["Core"][0].sha256 == "known" and found["Core"][0].branch["status"] == "identical" + assert any("releaseAssets(first:20,after:" in q["query"] for q in fetch.queries) + + +def test_local_inventory_and_github_reads_overlap(tmp_path, monkeypatch): + root = tmp_path / "main" + make_jar(root / "plugins", "core.jar", "Core", "1.0.0") + local_started, remote_started = threading.Event(), threading.Event() + original = inventory._local_server + def local(*args): + local_started.set() + assert remote_started.wait(3), "GitHub work waited for local inventory" + return original(*args) + def remote(path, body=None): + remote_started.set() + assert local_started.wait(3), "local inventory waited for GitHub" + return [] + monkeypatch.setattr(inventory, "_local_server", local) + found, servers = inventory.check_servers({"main": (root, True)}, remote) + assert found.error is None and servers["main"]["jars"][0].name == "Core" + + +@pytest.mark.parametrize("authenticated", [False, True]) +@pytest.mark.parametrize("releases_only", [False, True]) +def test_exhausted_history_with_missing_jar_digest_keeps_unmatched_bytes_unverified(tmp_path, monkeypatch, authenticated, releases_only): + if authenticated: + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + key = "main" if releases_only else "dev" + root = tmp_path / key + make_jar(root / "plugins", "core-custom.jar", "Core", "legacy") + old = release("v1.0.0", "2026-09-01T00:00:00Z", "core-1.0.0.jar", "unknown") + old["assets"][-1]["digest"] = None + fetch = Github({"Core": [release("v2.0.0", "2026-10-01T00:00:00Z", "core-2.0.0.jar", "new"), old]}) + found, local = inventory.check_servers({key: (root, releases_only)}, fetch) + [plugin], others = inventory.compare(found, local) + assert plugin["servers"][key]["state"] == "unverified" and not others + assert plugin["servers"][key]["release"] is None + assert "Core" not in found.complete + + +@pytest.mark.parametrize("authenticated", [False, True]) +def test_positive_historical_hash_wins_even_when_another_release_has_no_jar_digest(tmp_path, monkeypatch, authenticated): + if authenticated: + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + root = tmp_path / "main" + sha = make_jar(root / "plugins", "renamed-engine.jar", "Core", "legacy") + unknown = release("v1.5.0", "2026-09-15T00:00:00Z", "core-1.5.0.jar", "unknown") + unknown["assets"][-1]["digest"] = None + rows = [release("v2.0.0", "2026-10-01T00:00:00Z", "core-2.0.0.jar", "new"), unknown, + release("v1.0.0", "2026-09-01T00:00:00Z", "core-1.0.0.jar", sha)] + found, local = inventory.check_servers({"main": (root, True)}, Github({"Core": rows})) + [plugin], others = inventory.compare(found, local) + copy = plugin["servers"]["main"] + assert copy["state"] == "older_release" and copy["release"] == "1.0.0" and not others + assert plugin["latest"]["version"] == "2.0.0" + + +@pytest.mark.parametrize("authenticated", [False, True]) +def test_second_jar_asset_in_a_historical_release_can_prove_the_installed_release(tmp_path, monkeypatch, authenticated): + if authenticated: + monkeypatch.setenv("SERVERS_GITHUB_TOKEN", "token") + root = tmp_path / "main" + sha = make_jar(root / "plugins", "renamed-engine.jar", "Core", "legacy") + old = release("v1.0.0", "2026-09-01T00:00:00Z", "core-api-1.0.0.jar", "f" * 64) + old["assets"].append({"name": "core-1.0.0.jar", "digest": f"sha256:{sha}"}) + rows = [release("v2.0.0", "2026-10-01T00:00:00Z", "core-2.0.0.jar", "new"), old] + found, local = inventory.check_servers({"main": (root, True)}, Github({"Core": rows})) + [plugin], others = inventory.compare(found, local) + assert plugin["servers"]["main"]["state"] == "older_release" and not others + assert plugin["servers"]["main"]["release"] == "1.0.0" + assert sha in {r.sha256 for r in found["Core"]} + + +def test_http_body_trickle_cannot_extend_the_check_deadline(monkeypatch): + import http.server + import time + from contextlib import contextmanager + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + self.send_response(200) + self.end_headers() + try: + for byte in b'{"value":"a deliberately slow JSON body"}': + self.wfile.write(bytes([byte])) + self.wfile.flush() + time.sleep(.02) + except (BrokenPipeError, ConnectionResetError): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + original = inventory.urllib.request.urlopen + + @contextmanager + def local(_request, timeout): + with original(f"http://127.0.0.1:{server.server_port}/", timeout=timeout) as response: + yield response + + monkeypatch.setattr(inventory.urllib.request, "urlopen", local) + start = time.monotonic() + try: + with pytest.raises((TimeoutError, OSError)): + inventory._github("/graphql", timeout=.1) + assert time.monotonic() - start < .4 + finally: + server.shutdown() + server.server_close() + thread.join(2) diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 51a17801..505cd737 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -12,6 +12,7 @@ services: - SKINS_DEV=1 - STAFF_KEY=dev-staff-key - PLUGIN_KEY=dev-plugin-key + - SERVERS_GITHUB_TOKEN=${SERVERS_GITHUB_TOKEN:-} # Patch notes need Postgres. Staging keeps its own, so nothing here reaches production. - SUPABASE_DB_URL=postgresql://postgres:dev-patchnotes@patchnotes-db:5432/tfmc volumes: diff --git a/frontend/app/components/admin/ServersOverview.test.tsx b/frontend/app/components/admin/ServersOverview.test.tsx index 2243eae5..4fd82518 100644 --- a/frontend/app/components/admin/ServersOverview.test.tsx +++ b/frontend/app/components/admin/ServersOverview.test.tsx @@ -1,8 +1,9 @@ /** @vitest-environment jsdom */ -import { cleanup, fireEvent, render, screen, within } from "@testing-library/react"; +import { act, cleanup, fireEvent, render, screen, within } from "@testing-library/react"; import { afterEach, expect, it, vi } from "vitest"; import ServersOverview from "./ServersOverview"; +import * as ServerControls from "./ServerControls"; import { AccountApiError } from "../../../lib/account/api"; import { getServers, @@ -18,6 +19,7 @@ vi.mock("../../../lib/admin/servers", async (importOriginal) => ({ afterEach(() => { cleanup(); + vi.restoreAllMocks(); vi.mocked(getServers).mockReset(); }); @@ -33,7 +35,7 @@ const copy = (over: Partial = {}): PluginCopy => ({ ...over, }); -const latest = (version: string, branch: ServerPlugin["latest"]["branch"] = null) => ({ +const latest = (version: string, branch: NonNullable["branch"] = null) => ({ version, published_at: "2026-10-01T00:00:00Z", url: "https://github.com/x", @@ -45,7 +47,6 @@ const overview: Overview = { github: { status: "available", checked_at: "2026-10-11T18:00:00Z", - stale: false, }, servers: [ { @@ -167,7 +168,7 @@ it("shows every plugin, other plugins, and filters by name", async () => { it("says when a server or GitHub can't be read", async () => { vi.mocked(getServers).mockResolvedValue({ ...overview, - github: { status: "unavailable", checked_at: null, stale: false }, + github: { status: "unavailable", checked_at: null }, servers: [overview.servers[0], { ...overview.servers[1], status: "not_configured" }], plugins: [], others: [], @@ -182,3 +183,62 @@ it("shows the staff gate when the role is too low", async () => { render(); expect(await screen.findByText("This page is for TFMC staff only.")).toBeTruthy(); }); + +it("shows a failed fresh check without relabelling the previous result", async () => { + vi.mocked(getServers).mockResolvedValueOnce(overview).mockRejectedValueOnce(new Error("offline")); + render(); + await screen.findByText("= main"); + fireEvent.click(screen.getByRole("button", { name: "Check again" })); + expect(await screen.findByText(/Check failed/)).toBeTruthy(); + expect(screen.getByText("= main")).toBeTruthy(); + expect(getServers).toHaveBeenCalledTimes(2); +}); + +it("shows failed repository lookups and unverified jars as warnings", async () => { + vi.mocked(getServers).mockResolvedValue({ + ...overview, + github: { status: "partial", checked_at: overview.checked_at }, + plugins: [ + { repo: "Core", latest: null, error: "github_unavailable", servers: { main: copy({ state: "unverified" }) } }, + ], + }); + render(); + expect( + (await screen.findAllByText(/Unverified|Couldn.t check|couldn’t check|didn.t answer|didn’t answer/i)).length, + ).toBeGreaterThan(0); + expect(screen.getByText("Core")).toBeTruthy(); +}); + +it("keeps the newer check when an older request completes afterwards", async () => { + let refresh: (() => void) | undefined; + const original = ServerControls.useServerControls; + vi.spyOn(ServerControls, "useServerControls").mockImplementation((onChange) => { + refresh = onChange; + return original(onChange); + }); + let finishOlder!: (data: Overview) => void; + let finishNewer!: (data: Overview) => void; + const older = new Promise((resolve) => { + finishOlder = resolve; + }); + const newer = new Promise((resolve) => { + finishNewer = resolve; + }); + vi.mocked(getServers).mockResolvedValueOnce(overview).mockReturnValueOnce(older).mockReturnValueOnce(newer); + render(); + await screen.findByText("= main"); + act(() => { + refresh!(); + refresh!(); + }); + await act(async () => + finishNewer({ + ...overview, + plugins: [{ repo: "FreshResult", latest: latest("9.0.0"), servers: { main: copy({ state: "older_release" }) } }], + }), + ); + expect(await screen.findByText("FreshResult")).toBeTruthy(); + await act(async () => finishOlder(overview)); + expect(screen.getByText("FreshResult")).toBeTruthy(); + expect(screen.queryByText("Archaeo")).toBeNull(); +}); diff --git a/frontend/app/components/admin/ServersOverview.tsx b/frontend/app/components/admin/ServersOverview.tsx index 037c101f..db1943bb 100644 --- a/frontend/app/components/admin/ServersOverview.tsx +++ b/frontend/app/components/admin/ServersOverview.tsx @@ -1,7 +1,7 @@ "use client"; import LoadingSkeleton from "@/app/components/shared/LoadingSkeleton"; -import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { branchFlag, @@ -67,6 +67,7 @@ function ServerCard({ server, plugins, actions }: { server: ServerInfo; plugins: const available = server.status === "available"; const [startup, dot] = STARTUP[String(server.started)]; const c = counts(plugins, server.key); + const unverified = plugins.filter((plugin) => plugin.servers[server.key]?.state === "unverified").length; const stats: [number, string, Tone][] = [ server.releases_only ? [c.notReleases, "Not releases", "bad"] : [c.notReleases, "Test builds", "test"], [c.waiting, "Need a restart", "warn"], @@ -116,6 +117,11 @@ function ServerCard({ server, plugins, actions }: { server: ServerInfo; plugins: : "The site isn’t set up to read this server’s files."}

)} + {available && unverified ? ( +

+ {unverified} plugin{unverified === 1 ? "" : "s"} unverified +

+ ) : null}
{actions}
@@ -124,7 +130,14 @@ function ServerCard({ server, plugins, actions }: { server: ServerInfo; plugins: } /** The latest release, and whether it is the repository's default branch. */ -function LatestCell({ latest }: { latest: ServerPlugin["latest"] }) { +function LatestCell({ latest, error }: { latest: ServerPlugin["latest"]; error?: ServerPlugin["error"] }) { + if (!latest) { + return ( + + Unverified + + ); + } const mark = branchFlag(latest.branch); return (
@@ -147,6 +160,7 @@ function LatestCell({ latest }: { latest: ServerPlugin["latest"] }) { {mark.label} ) : null} + {error ? Unverified : null}
); } @@ -158,7 +172,7 @@ function checkedAt(iso: string): string { return at.toLocaleTimeString(undefined, { timeStyle: "short" }); } -function CopyCell({ copy, label, latest }: { copy: PluginCopy | undefined; label: string; latest: string }) { +function CopyCell({ copy, label, latest }: { copy: PluginCopy | undefined; label: string; latest: string | null }) { if (!copy) return
; const found = flags(copy); const pending = waiting(copy); @@ -190,25 +204,39 @@ export default function ServersOverview() { const [data, setData] = useState(null); const [gate, setGate] = useState(null); const [loading, setLoading] = useState(true); + const [checkFailed, setCheckFailed] = useState(false); + const requestSequence = useRef(0); const [view, setView] = useState("look"); const [query, setQuery] = useState(""); const load = useCallback(() => { + const sequence = ++requestSequence.current; setLoading(true); + setCheckFailed(false); getServers() .then((overview) => { + if (sequence !== requestSequence.current) return; setData(overview); setGate(null); }) - .catch((err) => setGate(gateKind(err))) - .finally(() => setLoading(false)); + .catch((err) => { + if (sequence !== requestSequence.current) return; + setCheckFailed(true); + setGate(gateKind(err)); + }) + .finally(() => { + if (sequence === requestSequence.current) setLoading(false); + }); }, []); const control = useServerControls(load); useEffect(() => { const start = setTimeout(load, 0); - return () => clearTimeout(start); + return () => { + clearTimeout(start); + requestSequence.current += 1; + }; }, [load]); const readable = useMemo(() => data?.servers.filter((s) => s.status === "available") ?? [], [data]); @@ -218,13 +246,23 @@ export default function ServersOverview() { }, [data]); if (gate && !data) return ; - if (!data) return ; + if (!data) + return ( + <> +

+ Checking servers and releases… +

+ + + ); const needle = query.trim().toLowerCase(); const rows = (view === "look" ? needsLook : data.plugins).filter((p) => p.repo.toLowerCase().includes(needle)); const others = data.others.filter((o) => o.name.toLowerCase().includes(needle)); const columns = COLUMNS[readable.length] ?? COLUMNS[2]; const releaseColumns = RELEASE_COLUMNS[readable.length] ?? RELEASE_COLUMNS[2]; + const unverifiedOthers = data.others.some((other) => other.unverified); + const incomplete = checkFailed || data.github.status !== "available" || unverifiedOthers; const tabs: [View, string, number][] = [ ["look", "Needs a look", needsLook.length], ["all", "TFMC", data.plugins.length], @@ -239,6 +277,11 @@ export default function ServersOverview() { {loading ? "Checking…" : "Check again"}

+ {checkFailed ? ( +

+ Check failed. Showing the previous results from {formatLocal(data.checked_at)}. Try checking again. +

+ ) : null}
{data.servers.map((server) => ( @@ -257,9 +300,17 @@ export default function ServersOverview() {

GitHub didn’t answer, so releases can’t be compared. Try again in a minute.

- ) : data.github.stale ? ( -

- GitHub didn’t answer; releases are from {formatLocal(data.github.checked_at)}. + ) : data.github.status === "partial" ? ( +

+ Some GitHub checks failed. Plugins marked Unverified need another check. +

+ ) : data.github.status === "busy" ? ( +

+ Another release check is running. Try checking again shortly. +

+ ) : unverifiedOthers ? ( +

+ Some installed files couldn’t be verified. Check plugins marked Unverified in Other plugins.

) : null} @@ -319,6 +370,9 @@ export default function ServersOverview() { {other.differs ? ( Differs ) : null} + {other.unverified ? ( + Unverified + ) : null}

{readable.map((s) => { const found = other.servers[s.key]; @@ -356,17 +410,24 @@ export default function ServersOverview() { >

{plugin.repo}

- +
{readable.map((s) => ( - + ))} ))} {!rows.length ? (
  • {view === "look" && !needle - ? "Every server runs the latest release of every plugin." + ? incomplete + ? "Release checks are incomplete. Check again to verify these plugins." + : "Every server runs the latest release of every plugin." : "No plugins match."}
    • ) : null} diff --git a/frontend/lib/admin/servers.test.ts b/frontend/lib/admin/servers.test.ts index a1011b61..ab7f8989 100644 --- a/frontend/lib/admin/servers.test.ts +++ b/frontend/lib/admin/servers.test.ts @@ -95,3 +95,9 @@ describe("latest release against the default branch", () => { expect(branchFlag(branch("diverged", 2))?.label).toBe("Off main"); }); }); + +it("keeps unverified data visible without claiming it is a test build", () => { + const unknown = copy({ state: "unverified" }); + expect(flags(unknown)).toContainEqual({ label: "Unverified", tone: "warn" }); + expect(severity({ repo: "Core", latest: null, error: "github_unavailable", servers: { main: unknown } })).toBe(1); +}); diff --git a/frontend/lib/admin/servers.ts b/frontend/lib/admin/servers.ts index dd0392fe..716d4e08 100644 --- a/frontend/lib/admin/servers.ts +++ b/frontend/lib/admin/servers.ts @@ -15,7 +15,7 @@ export type ServerInfo = { }; export type PluginCopy = { - state: "latest" | "older_release" | "not_release" | "test_build" | "missing"; + state: "latest" | "older_release" | "not_release" | "test_build" | "unverified" | "missing"; files: string[]; /** The jar's own version (plugin.yml). */ version: string | null; @@ -40,22 +40,23 @@ export type ReleaseBranch = { export type ServerPlugin = { repo: string; - latest: { version: string; published_at: string; url: string; branch: ReleaseBranch | null }; + latest: { version: string; published_at: string; url: string; branch: ReleaseBranch | null } | null; + error?: "github_unavailable" | "unverified"; servers: Partial>; }; export type OtherPlugin = { name: string; differs: boolean; + unverified?: boolean; servers: Partial>; }; export type ServersOverview = { checked_at: string; github: { - status: "available" | "unavailable"; + status: "available" | "partial" | "unavailable" | "busy"; checked_at: string | null; - stale: boolean; }; servers: ServerInfo[]; plugins: ServerPlugin[]; @@ -88,6 +89,7 @@ export function flags(copy: PluginCopy): Flag[] { if (copy.state === "not_release") found.push({ label: "Not a release", tone: "bad" }); if (copy.state === "test_build") found.push({ label: "Test build", tone: "test" }); if (copy.state === "older_release") found.push({ label: "Older release", tone: "warn" }); + if (copy.state === "unverified") found.push({ label: "Unverified", tone: "warn" }); if (copy.duplicate) found.push({ label: "Two jars", tone: "bad" }); if (copy.loaded_state === "not_loaded") found.push({ label: "Didn’t start", tone: "bad" }); if (waiting(copy)) found.push({ label: "Needs restart", tone: "warn" }); @@ -107,6 +109,7 @@ export function severity(plugin: ServerPlugin): number | null { .flatMap((copy) => (copy ? flags(copy) : [])) .filter((flag) => flag.tone !== "none") .map((flag) => RANK[flag.tone]); + if (!plugin.latest || plugin.error) tones.push(RANK.warn); return tones.length ? Math.min(...tones) : null; } From 58130d65f25a00fa362f5996fb965229cc09b76b Mon Sep 17 00:00:00 2001 From: Ryan Barlow <7389646+ryanbarlow97@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:37:47 +0000 Subject: [PATCH 2/3] fix(servers): load Dev token through its environment file --- docker-compose.staging.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 505cd737..06f2b974 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -5,6 +5,9 @@ services: build: context: . dockerfile: backend/Dockerfile + env_file: + - path: .env + required: false ports: # Host 8001 is often taken; use 18001 for staging - "18001:8000" @@ -12,7 +15,6 @@ services: - SKINS_DEV=1 - STAFF_KEY=dev-staff-key - PLUGIN_KEY=dev-plugin-key - - SERVERS_GITHUB_TOKEN=${SERVERS_GITHUB_TOKEN:-} # Patch notes need Postgres. Staging keeps its own, so nothing here reaches production. - SUPABASE_DB_URL=postgresql://postgres:dev-patchnotes@patchnotes-db:5432/tfmc volumes: From 98b0972a5a9772a6d1c2a495f86f649e8c8936d7 Mon Sep 17 00:00:00 2001 From: Ryan Barlow <7389646+ryanbarlow97@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:42:52 +0000 Subject: [PATCH 3/3] fix(servers): keep host-managed Dev Compose configuration intact --- docker-compose.staging.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 06f2b974..51a17801 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -5,9 +5,6 @@ services: build: context: . dockerfile: backend/Dockerfile - env_file: - - path: .env - required: false ports: # Host 8001 is often taken; use 18001 for staging - "18001:8000"