From 26dc15478e030c0a2da236c99941bad33a0939a1 Mon Sep 17 00:00:00 2001 From: Mike Rosseel Date: Sat, 4 Jul 2026 13:48:18 +0200 Subject: [PATCH 1/5] fix(migration): generate NM wifi keyfiles in Python pre-reboot with correct SSID encoding --- python/PiFinder/nixos_migration_wifi.py | 276 +++++++++++++++++++ python/scripts/nixos_migration.sh | 15 + python/scripts/nixos_migration_init.sh | 109 +------- python/tests/test_nixos_migration_wifi.py | 321 ++++++++++++++++++++++ 4 files changed, 623 insertions(+), 98 deletions(-) create mode 100644 python/PiFinder/nixos_migration_wifi.py create mode 100644 python/tests/test_nixos_migration_wifi.py diff --git a/python/PiFinder/nixos_migration_wifi.py b/python/PiFinder/nixos_migration_wifi.py new file mode 100644 index 00000000..03b0e688 --- /dev/null +++ b/python/PiFinder/nixos_migration_wifi.py @@ -0,0 +1,276 @@ +"""Convert wpa_supplicant.conf to NetworkManager keyfiles. + +Runs during the pre-migration phase on Debian, before reboot into the +initramfs. Keyfiles get staged into the initramfs build dir and the +init script just copies them into the new rootfs — much safer than +generating them in busybox shell after the rootfs has been formatted. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +import uuid +from pathlib import Path +from typing import Callable, Iterable, List, Optional + + +WPA_NETWORK_OPEN = re.compile(r"^\s*network\s*=\s*\{") +WPA_NETWORK_CLOSE = re.compile(r"^\s*\}") +WPA_KEY_VALUE = re.compile(r"^\s*([a-zA-Z0-9_]+)\s*=\s*(.*?)\s*$") + +HEX_PSK_RE = re.compile(r"^[0-9a-fA-F]{64}$") +HEX_STRING_RE = re.compile(r"^(?:[0-9a-fA-F]{2})+$") + + +class Network: + __slots__ = ("ssid", "psk") + + def __init__(self, ssid: str, psk: Optional[str]) -> None: + self.ssid = ssid + self.psk = psk + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, Network) + and self.ssid == other.ssid + and self.psk == other.psk + ) + + def __repr__(self) -> str: + psk_repr = "None" if self.psk is None else f"<{len(self.psk)}c>" + return f"Network(ssid={self.ssid!r}, psk={psk_repr})" + + +def _unquote(value: str) -> str: + """Strip a single surrounding pair of double quotes if present.""" + if len(value) >= 2 and value.startswith('"') and value.endswith('"'): + return value[1:-1] + return value + + +def _parse_ssid(value: str) -> str: + """Decode a wpa_supplicant ssid value. + + Quoted values are plain strings. Unquoted values are hex-encoded byte + strings per wpa_supplicant syntax — decode them here, or the network + name ends up mangled; surrogateescape keeps non-UTF-8 SSID bytes + round-trippable into the keyfile byte list. + """ + if len(value) >= 2 and value.startswith('"') and value.endswith('"'): + return value[1:-1] + if HEX_STRING_RE.fullmatch(value): + return bytes.fromhex(value).decode("utf-8", "surrogateescape") + return value + + +def parse_wpa_supplicant_conf(text: str) -> List[Network]: + """Parse the subset of wpa_supplicant.conf we care about. + + Recognises `network={ ... }` blocks containing `ssid=` and `psk=`. + Quoted values get their outer quotes stripped. Unquoted PSKs (the + 64-hex-char pre-shared-key form) are kept verbatim — NetworkManager + accepts both. Networks without an SSID are skipped. + + Returns the networks in declaration order. + """ + networks: List[Network] = [] + in_net = False + ssid: Optional[str] = None + psk: Optional[str] = None + + for raw_line in text.splitlines(): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + + if WPA_NETWORK_OPEN.match(line): + in_net = True + ssid = None + psk = None + continue + + if WPA_NETWORK_CLOSE.match(line): + if in_net and ssid is not None: + networks.append(Network(ssid=ssid, psk=psk)) + in_net = False + ssid = None + psk = None + continue + + if not in_net: + continue + + match = WPA_KEY_VALUE.match(line) + if not match: + continue + key, value = match.group(1), match.group(2) + if key == "ssid": + ssid = _parse_ssid(value) + elif key == "psk": + psk = _unquote(value) + + return networks + + +def ssid_to_bytelist(ssid: str) -> str: + """Encode an SSID as a NetworkManager keyfile byte list (`97;112;...`). + + NM's keyfile format documents exactly two ssid forms: a plain string and + a semicolon-separated list of DECIMAL byte values. Anything else (hex, + 0x-prefixed or not) is silently kept as a literal-string SSID, mangling + the network name so the device can never join it. surrogateescape + restores non-UTF-8 bytes captured from wpa_supplicant's hex ssid form. + """ + return "".join(f"{b};" for b in ssid.encode("utf-8", "surrogateescape")) + + +def escape_keyfile_value(value: str) -> str: + """Escape characters that have meaning in NM keyfile values. + + Backslash and semicolon are the only special characters in a plain + string value. (The byte-list format uses the semicolon as separator, + but we don't use that for the PSK.) + """ + return value.replace("\\", "\\\\").replace(";", "\\;") + + +_SAFE_FN_CHARS = re.compile(r"[^A-Za-z0-9._-]") + + +def sanitize_filename(ssid: str) -> str: + """Build a safe filename for a connection keyfile from an SSID. + + Non-alphanumeric characters (except `.`, `_`, `-`) are replaced with + `_`. The result is also guarded against the empty string and `.`/`..` + so a hostile or odd SSID can't escape the connections directory. + """ + cleaned = _SAFE_FN_CHARS.sub("_", ssid) + if cleaned in ("", ".", ".."): + return "wifi" + return cleaned + + +def build_keyfile( + ssid: str, + psk: Optional[str], + connection_uuid: Optional[str] = None, +) -> str: + """Build a NetworkManager keyfile body for a single WiFi connection. + + `psk=None` produces an open-network keyfile (no [wifi-security]). + `connection_uuid=None` generates a fresh v4 UUID. + """ + if connection_uuid is None: + connection_uuid = str(uuid.uuid4()) + + id_escaped = escape_keyfile_value(ssid) + ssid_encoded = ssid_to_bytelist(ssid) + + lines = [ + "[connection]", + f"id={id_escaped}", + f"uuid={connection_uuid}", + "type=wifi", + "autoconnect=true", + "", + "[wifi]", + "mode=infrastructure", + f"ssid={ssid_encoded}", + "", + ] + + if psk is not None and psk != "": + psk_escaped = escape_keyfile_value(psk) + lines.extend( + [ + "[wifi-security]", + "key-mgmt=wpa-psk", + f"psk={psk_escaped}", + "", + ] + ) + + lines.extend( + [ + "[ipv4]", + "method=auto", + "", + "[ipv6]", + "method=auto", + "", + ] + ) + return "\n".join(lines) + + +def emit_keyfiles( + networks: Iterable[Network], + output_dir: Path, + uuid_factory: Callable[[], str] = lambda: str(uuid.uuid4()), +) -> List[Path]: + """Write a keyfile per network into `output_dir`. + + Filenames are based on a sanitised SSID; collisions get a numeric + suffix. Files are mode 0600. Returns the list of paths written. + """ + output_dir.mkdir(parents=True, exist_ok=True) + written: List[Path] = [] + used: set = set() + + for net in networks: + base = sanitize_filename(net.ssid) + name = base + n = 1 + while name in used: + n += 1 + name = f"{base}_{n}" + used.add(name) + + path = output_dir / f"{name}.nmconnection" + body = build_keyfile(net.ssid, net.psk, connection_uuid=uuid_factory()) + path.write_text(body) + os.chmod(path, 0o600) + written.append(path) + + return written + + +def _main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Convert wpa_supplicant.conf to NetworkManager keyfiles for " + "the PiFinder NixOS migration." + ) + ) + parser.add_argument( + "--wpa-conf", + required=True, + help="Path to the source wpa_supplicant.conf file.", + ) + parser.add_argument( + "--out", + required=True, + help="Directory to write the .nmconnection files into (created if absent).", + ) + args = parser.parse_args(argv) + + src = Path(args.wpa_conf) + if not src.exists(): + print(f"wpa_supplicant.conf not found at {src}", file=sys.stderr) + return 0 + + networks = parse_wpa_supplicant_conf(src.read_text()) + if not networks: + print(f"No networks parsed from {src}", file=sys.stderr) + return 0 + + written = emit_keyfiles(networks, Path(args.out)) + print(f"Wrote {len(written)} keyfile(s) to {args.out}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/python/scripts/nixos_migration.sh b/python/scripts/nixos_migration.sh index 74092831..3aea219d 100755 --- a/python/scripts/nixos_migration.sh +++ b/python/scripts/nixos_migration.sh @@ -209,6 +209,21 @@ fi cp "${INIT_SCRIPT}" "${INITRAMFS_DIR}/init" chmod +x "${INITRAMFS_DIR}/init" +# Pre-stage NetworkManager keyfiles from the live wpa_supplicant.conf. +# Generating them here (with Python, on the full Debian system) rather +# than in the busybox initramfs lets us unit-test the conversion. The +# init script just copies these into the new rootfs. +WIFI_STAGED_DIR="${INITRAMFS_DIR}/wifi-staged" +WPA_CONF="/etc/wpa_supplicant/wpa_supplicant.conf" +if [ -f "${WPA_CONF}" ]; then + PIFINDER_PYTHON_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + PYTHONPATH="${PIFINDER_PYTHON_ROOT}" python3 \ + -m PiFinder.nixos_migration_wifi \ + --wpa-conf "${WPA_CONF}" \ + --out "${WIFI_STAGED_DIR}" \ + || fail 5 "WiFi keyfile generation failed" +fi + # Metadata: paths + sizes so init script knows where to find things cat > "${INITRAMFS_DIR}/migration_meta" </dev/null || true fi +# Any NM keyfiles the user already had on Debian — preserved as-is. NM_SRC="${MOUNT_ROOT}/etc/NetworkManager/system-connections" if [ -d "${NM_SRC}" ]; then - mkdir -p /tmp/wifi/nm-connections cp -a "${NM_SRC}/." /tmp/wifi/nm-connections/ 2>/dev/null || true fi @@ -317,101 +320,11 @@ show 70 "Migrating WiFi" NM_DIR="${MOUNT_NEW}/etc/NetworkManager/system-connections" mkdir -p "${NM_DIR}" -# NM keyfile helpers — SSID becomes hex bytes (safe for any content), -# PSK gets minimal keyfile escaping (backslash + semicolon), filename -# is sanitized so a hostile or unusual SSID can't escape NM_DIR. -ssid_to_hex() { - printf '%s' "$1" | od -An -tx1 | tr -d ' \n' | sed 's/\(..\)/\1;/g' -} -escape_kf() { - printf '%s' "$1" | sed -e 's/\\/\\\\/g' -e 's/;/\\;/g' -} -sanitize_fn() { - printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_' -} - -if [ -f /tmp/wifi/wpa_supplicant.conf ]; then - SSID="" - PSK="" - IN_NET=0 - - while IFS= read -r line; do - line=$(printf '%s' "${line}" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//') - - case "${line}" in - network=*) - IN_NET=1 - SSID="" - PSK="" - ;; - "}") - if [ "${IN_NET}" = "1" ] && [ -n "${SSID}" ]; then - SSID_HEX=$(ssid_to_hex "${SSID}") - ID_ESC=$(escape_kf "${SSID}") - FN=$(sanitize_fn "${SSID}") - # Guard against empty/dotfile filenames after sanitization - case "${FN}" in - ""|.|..) FN="wifi" ;; - esac - NM_FILE="${NM_DIR}/${FN}.nmconnection" - - if [ -n "${PSK}" ]; then - PSK_ESC=$(escape_kf "${PSK}") - cat > "${NM_FILE}" < "${NM_FILE}" </dev/null || true + chmod 600 "${NM_DIR}"/*.nmconnection 2>/dev/null || true fi sync diff --git a/python/tests/test_nixos_migration_wifi.py b/python/tests/test_nixos_migration_wifi.py new file mode 100644 index 00000000..9b621fb4 --- /dev/null +++ b/python/tests/test_nixos_migration_wifi.py @@ -0,0 +1,321 @@ +import re + +import pytest + +from PiFinder.nixos_migration_wifi import ( + Network, + _parse_ssid, + build_keyfile, + emit_keyfiles, + escape_keyfile_value, + parse_wpa_supplicant_conf, + sanitize_filename, + ssid_to_bytelist, +) + + +UUID_V4_RE = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" +) + + +@pytest.mark.unit +class TestParseWpaSupplicantConf: + def test_empty(self): + assert parse_wpa_supplicant_conf("") == [] + + def test_single_wpa_network(self): + conf = """ + ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev + update_config=1 + country=BE + + network={ + ssid="APME" + psk="hunter12" + } + """ + assert parse_wpa_supplicant_conf(conf) == [Network("APME", "hunter12")] + + def test_open_network_has_no_psk(self): + conf = """ + network={ + ssid="OpenNet" + key_mgmt=NONE + } + """ + assert parse_wpa_supplicant_conf(conf) == [Network("OpenNet", None)] + + def test_multiple_networks_preserve_order(self): + conf = """ + network={ + ssid="first" + psk="pw1" + } + network={ + ssid="second" + psk="pw2" + } + network={ + ssid="third" + psk="pw3" + } + """ + nets = parse_wpa_supplicant_conf(conf) + assert [n.ssid for n in nets] == ["first", "second", "third"] + + def test_hex_psk_preserved_verbatim(self): + hex_psk = "a" * 64 + conf = f""" + network={{ + ssid="HexPsk" + psk={hex_psk} + }} + """ + result = parse_wpa_supplicant_conf(conf) + assert result == [Network("HexPsk", hex_psk)] + + def test_unquoted_ssid_kept(self): + # Some configs use unquoted SSIDs for short alphanumerics. + conf = """ + network={ + ssid=plain + psk="pw" + } + """ + assert parse_wpa_supplicant_conf(conf) == [Network("plain", "pw")] + + def test_ssid_with_special_chars(self): + conf = """ + network={ + ssid="0x20" + psk="pw" + } + network={ + ssid="hackerspace.gent" + psk="pw2" + } + """ + nets = parse_wpa_supplicant_conf(conf) + assert nets == [ + Network("0x20", "pw"), + Network("hackerspace.gent", "pw2"), + ] + + def test_network_without_ssid_skipped(self): + conf = """ + network={ + psk="orphan" + } + network={ + ssid="valid" + psk="pw" + } + """ + assert parse_wpa_supplicant_conf(conf) == [Network("valid", "pw")] + + def test_comments_and_blank_lines_ignored(self): + conf = """ + # global comment + ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev + + network={ + # inner comment + ssid="commented" + psk="pw" # trailing comment + } + """ + assert parse_wpa_supplicant_conf(conf) == [Network("commented", "pw")] + + +@pytest.mark.unit +class TestSsidToBytelist: + def test_ascii(self): + assert ssid_to_bytelist("APME") == "65;80;77;69;" + + def test_bytes_are_decimal(self): + # The whole point of this function — NM only parses DECIMAL byte + # lists; hex (with or without 0x prefix) is silently kept as a + # literal-string SSID and the network name is mangled. + assert ssid_to_bytelist("apollo") == "97;112;111;108;108;111;" + + def test_utf8_bytes(self): + # SSID can contain non-ASCII; should be encoded as utf-8 bytes + assert ssid_to_bytelist("é") == "195;169;" + + def test_empty(self): + assert ssid_to_bytelist("") == "" + + def test_non_utf8_bytes_round_trip(self): + # A hex wpa ssid holding non-UTF-8 bytes survives parse -> encode. + assert ssid_to_bytelist(_parse_ssid("ff00")) == "255;0;" + + +@pytest.mark.unit +class TestParseSsidValue: + def test_quoted_is_plain_string(self): + assert _parse_ssid('"apollo"') == "apollo" + + def test_unquoted_hex_is_decoded(self): + # wpa_supplicant stores SSIDs with special characters as unquoted + # hex strings; these must be decoded, not used as the name. + assert _parse_ssid("61706f6c6c6f") == "apollo" + + def test_quoted_hex_lookalike_stays_verbatim(self): + # A network genuinely NAMED like a hex string is quoted in + # wpa_supplicant, so it must not be decoded. + assert _parse_ssid('"61706f"') == "61706f" + + def test_non_hex_unquoted_stays_verbatim(self): + assert _parse_ssid("abc") == "abc" + + def test_hex_ssid_end_to_end(self): + conf = """ + network={ + ssid=61706f6c6c6f + psk="hunter12" + } + """ + assert parse_wpa_supplicant_conf(conf) == [Network("apollo", "hunter12")] + + +@pytest.mark.unit +class TestEscapeKeyfileValue: + def test_plain(self): + assert escape_keyfile_value("hunter12") == "hunter12" + + def test_semicolon_escaped(self): + assert escape_keyfile_value("a;b") == "a\\;b" + + def test_backslash_escaped(self): + assert escape_keyfile_value("a\\b") == "a\\\\b" + + def test_backslash_before_semicolon(self): + # Backslash must be escaped first so we don't double-escape the + # semicolon escape sequence we just produced. + assert escape_keyfile_value("\\;") == "\\\\\\;" + + +@pytest.mark.unit +class TestSanitizeFilename: + def test_plain(self): + assert sanitize_filename("APME") == "APME" + + def test_dot_preserved(self): + assert sanitize_filename("hackerspace.gent") == "hackerspace.gent" + + def test_slash_replaced(self): + assert sanitize_filename("a/b") == "a_b" + + def test_pathy_chars_replaced(self): + assert sanitize_filename("../etc/passwd") == ".._etc_passwd" + + def test_empty_becomes_wifi(self): + assert sanitize_filename("") == "wifi" + + def test_dot_becomes_wifi(self): + assert sanitize_filename(".") == "wifi" + + def test_dotdot_becomes_wifi(self): + assert sanitize_filename("..") == "wifi" + + +@pytest.mark.unit +class TestBuildKeyfile: + def test_contains_required_sections(self): + body = build_keyfile("APME", "hunter12", connection_uuid="fixed-uuid") + assert "[connection]" in body + assert "[wifi]" in body + assert "[wifi-security]" in body + assert "[ipv4]" in body + assert "[ipv6]" in body + + def test_uuid_present(self): + body = build_keyfile("APME", "pw", connection_uuid="abc-123") + assert "uuid=abc-123" in body + + def test_uuid_generated_when_not_provided(self): + body = build_keyfile("APME", "pw") + match = re.search(r"^uuid=(.+)$", body, re.MULTILINE) + assert match + assert UUID_V4_RE.match(match.group(1)) + + def test_ssid_encoded_as_decimal_bytelist(self): + body = build_keyfile("APME", "pw", connection_uuid="x") + assert "ssid=65;80;77;69;" in body + + def test_open_network_omits_security(self): + body = build_keyfile("OpenNet", None, connection_uuid="x") + assert "[wifi-security]" not in body + assert "key-mgmt" not in body + assert "psk=" not in body + + def test_empty_psk_treated_as_open(self): + body = build_keyfile("OpenNet", "", connection_uuid="x") + assert "[wifi-security]" not in body + + def test_psk_with_semicolon_escaped(self): + body = build_keyfile("S", "p;w", connection_uuid="x") + assert "psk=p\\;w" in body + + def test_psk_with_backslash_escaped(self): + body = build_keyfile("S", "p\\w", connection_uuid="x") + assert "psk=p\\\\w" in body + + def test_ipv4_method_auto(self): + body = build_keyfile("S", "pw", connection_uuid="x") + assert "[ipv4]\nmethod=auto" in body + + def test_id_uses_ssid(self): + body = build_keyfile("MyNet", "pw", connection_uuid="x") + assert "id=MyNet" in body + + +@pytest.mark.unit +class TestEmitKeyfiles: + def test_writes_one_file_per_network(self, tmp_path): + nets = [Network("a", "pw"), Network("b", None), Network("c", "pw3")] + written = emit_keyfiles(nets, tmp_path) + assert len(written) == 3 + assert {p.name for p in written} == { + "a.nmconnection", + "b.nmconnection", + "c.nmconnection", + } + + def test_file_mode_is_600(self, tmp_path): + emit_keyfiles([Network("a", "pw")], tmp_path) + mode = (tmp_path / "a.nmconnection").stat().st_mode & 0o777 + assert mode == 0o600 + + def test_creates_output_dir(self, tmp_path): + target = tmp_path / "nested" / "wifi" + emit_keyfiles([Network("a", "pw")], target) + assert (target / "a.nmconnection").exists() + + def test_collision_suffix(self, tmp_path): + # Two SSIDs that sanitize to the same filename + nets = [Network("a/b", "pw"), Network("a.b", "pw")] + # sanitize: "a/b" -> "a_b", "a.b" -> "a.b" (different) — pick another pair + nets = [Network("a/b", "pw"), Network("a;b", "pw")] + # Both sanitize to "a_b" + written = emit_keyfiles(nets, tmp_path) + names = sorted(p.name for p in written) + assert names == ["a_b.nmconnection", "a_b_2.nmconnection"] + + def test_each_file_gets_unique_uuid(self, tmp_path): + nets = [Network("a", "pw"), Network("b", "pw")] + emit_keyfiles(nets, tmp_path) + u1 = (tmp_path / "a.nmconnection").read_text() + u2 = (tmp_path / "b.nmconnection").read_text() + uuid1 = re.search(r"^uuid=(.+)$", u1, re.MULTILINE).group(1) + uuid2 = re.search(r"^uuid=(.+)$", u2, re.MULTILINE).group(1) + assert uuid1 != uuid2 + assert UUID_V4_RE.match(uuid1) + assert UUID_V4_RE.match(uuid2) + + def test_deterministic_with_injected_uuid(self, tmp_path): + nets = [Network("a", "pw")] + emit_keyfiles(nets, tmp_path, uuid_factory=lambda: "fixed-uuid-1234") + body = (tmp_path / "a.nmconnection").read_text() + assert "uuid=fixed-uuid-1234" in body From b0af19d2d1f20817df8d4a81433b5d9c796265f7 Mon Sep 17 00:00:00 2001 From: Mike Rosseel Date: Sat, 4 Jul 2026 15:58:38 +0200 Subject: [PATCH 2/5] =?UTF-8?q?fix(ci):=20restore=20the=20live=20Attic=20c?= =?UTF-8?q?ache=20key=20(8UU)=20=E2=80=94=20Vkem=20is=20the=20rolled-back?= =?UTF-8?q?=20rotation=20and=20verifies=20nothing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/nixos-pr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nixos-pr-build.yml b/.github/workflows/nixos-pr-build.yml index c8c4c181..19e74b44 100644 --- a/.github/workflows/nixos-pr-build.yml +++ b/.github/workflows/nixos-pr-build.yml @@ -121,7 +121,7 @@ jobs: extra-conf: | extra-system-features = big-parallel extra-substituters = https://cache.pifinder.eu/pifinder - extra-trusted-public-keys = pifinder:VkemNaMqXDcsYlpONItSvOOcBIa1vEfnpyqdetr3gck= + extra-trusted-public-keys = pifinder:8UU/O3oLkaJHHUyqEcPGl+9F1m4MqDca39Ewl49jBmE= - name: Attic login for push env: From e7907fe05d6c6c96ddcca61b87bef65cb0783a18 Mon Sep 17 00:00:00 2001 From: Mike Rosseel Date: Sun, 5 Jul 2026 13:46:08 +0200 Subject: [PATCH 3/5] =?UTF-8?q?fix(migration):=20keyfiles=20must=20be=20ro?= =?UTF-8?q?ot-owned=20=E2=80=94=20NM=20refuses=20'insecure'=20owner-1000?= =?UTF-8?q?=20files=20(pre-stage=20runs=20as=20the=20app=20user)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/scripts/nixos_migration_init.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/scripts/nixos_migration_init.sh b/python/scripts/nixos_migration_init.sh index 95623685..fc073f4e 100755 --- a/python/scripts/nixos_migration_init.sh +++ b/python/scripts/nixos_migration_init.sh @@ -324,6 +324,10 @@ mkdir -p "${NM_DIR}" # consolidated into /tmp/wifi/nm-connections during Phase 2. if [ -d /tmp/wifi/nm-connections ]; then cp -a /tmp/wifi/nm-connections/. "${NM_DIR}/" 2>/dev/null || true + # The pre-stage Python runs as the app user on the old OS, and cp -a + # preserves that owner — NetworkManager refuses keyfiles not owned by + # root ("File owner (1000) is insecure"). We are root here; make them so. + chown -R 0:0 "${NM_DIR}" 2>/dev/null || true chmod 600 "${NM_DIR}"/*.nmconnection 2>/dev/null || true fi From 3f7693322bf9aa86de73332c423020c3598e9cea Mon Sep 17 00:00:00 2001 From: Mike Rosseel Date: Sun, 5 Jul 2026 11:08:49 +0200 Subject: [PATCH 4/5] =?UTF-8?q?fix(migration):=20survive=20oversized=20tar?= =?UTF-8?q?balls=20=E2=80=94=20raise=20tmpfs=20cap,=20pre-reboot=20RAM=20f?= =?UTF-8?q?easibility=20check,=20auto-restore=20Raspbian=20on=20pre-format?= =?UTF-8?q?=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/scripts/nixos_migration.sh | 13 +++++++++- python/scripts/nixos_migration_init.sh | 35 +++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/python/scripts/nixos_migration.sh b/python/scripts/nixos_migration.sh index 3aea219d..e1265538 100755 --- a/python/scripts/nixos_migration.sh +++ b/python/scripts/nixos_migration.sh @@ -148,7 +148,18 @@ progress 68 "Preparing" TARBALL_SIZE=$(stat -c%s "${TARBALL}") -progress 75 "Tarball: $((TARBALL_SIZE / 1048576))MB" +# Feasibility gate — fail HERE, on the running OS, never in the initramfs. +# The initramfs copies the tarball to RAM (tmpfs) and needs headroom for the +# user-data backup and tools, so the tarball plus 400MB must fit inside +# total RAM. A 2GB board tops out around a ~1.4GB tarball. +MEM_TOTAL_MB=$(awk '/MemTotal/ {print int($2 / 1024)}' /proc/meminfo) +TARBALL_MB=$((TARBALL_SIZE / 1048576)) +NEEDED_MB=$((TARBALL_MB + 400)) +if [ "${NEEDED_MB}" -gt "${MEM_TOTAL_MB}" ]; then + fail 6 "Tarball too large for this board's RAM: needs ${NEEDED_MB}MB, have ${MEM_TOTAL_MB}MB" +fi + +progress 75 "Tarball: ${TARBALL_MB}MB" # --- Phase 5: Build initramfs --- progress 78 "Building initramfs" diff --git a/python/scripts/nixos_migration_init.sh b/python/scripts/nixos_migration_init.sh index fc073f4e..c0679b31 100755 --- a/python/scripts/nixos_migration_init.sh +++ b/python/scripts/nixos_migration_init.sh @@ -21,7 +21,10 @@ trap '' PIPE mount -t proc proc /proc 2>/dev/null || true mount -t sysfs sysfs /sys 2>/dev/null || true mount -t devtmpfs devtmpfs /dev 2>/dev/null || true -mount -t tmpfs tmpfs /tmp 2>/dev/null || true +# size=90%: the default tmpfs cap (50% of RAM) is smaller than the tarball on +# 2GB boards even when total RAM suffices — the explicit MemAvailable checks +# below are the real guard, the cap must never trip first. +mount -t tmpfs -o size=90% tmpfs /tmp 2>/dev/null || true # Load SPI modules for OLED progress display if [ -f /lib/modules/spi-bcm2835.ko ]; then @@ -70,12 +73,40 @@ show() { fi } +# Set to 1 immediately before the first destructive step (formatting). While +# it is 0, a failure can and must send the device back to the old OS. +DESTRUCTIVE=0 + fail() { if [ "${PROGRESS_READY}" -eq 1 ]; then echo "0 0 0 FAILED: $1" >&3 2>/dev/null || true fi echo "[FAILED] $1" echo "MIGRATION FAILED: $1" > /dev/console 2>/dev/null || true + + if [ "${DESTRUCTIVE}" = "0" ]; then + # Nothing has been formatted yet: restore the old OS's boot config and + # reboot into it, instead of stranding a headless device in a debug + # shell. The pre-migration script keeps .premigration backups. + echo "No data touched yet — restoring previous OS boot config..." + if [ "${PROGRESS_READY}" -eq 1 ]; then + echo "0 0 0 FAILED: $1 - rebooting to old OS" >&3 2>/dev/null || true + fi + mkdir -p /mnt/bootfix + if mount -t vfat "${BOOT_DEV}" /mnt/bootfix 2>/dev/null; then + [ -f /mnt/bootfix/config.txt.premigration ] && \ + cp /mnt/bootfix/config.txt.premigration /mnt/bootfix/config.txt + [ -f /mnt/bootfix/cmdline.txt.premigration ] && \ + cp /mnt/bootfix/cmdline.txt.premigration /mnt/bootfix/cmdline.txt + rm -f /mnt/bootfix/nixos_migration + sync + umount /mnt/bootfix + sleep 5 + reboot -f + fi + echo "Could not restore boot config — falling through to debug shell" + fi + echo "Dropping to shell for debugging..." exec /bin/sh } @@ -243,6 +274,8 @@ echo ", +" | sfdisk -N 2 "${SD_DEV}" --no-reread 2>/dev/null || true blockdev --rereadpt "${SD_DEV}" 2>/dev/null || true sleep 1 +# Point of no return: from here on a failure cannot go back to the old OS. +DESTRUCTIVE=1 show 50 "Formatting boot" mkfs.vfat -F 32 -n FIRMWARE "${BOOT_DEV}" || fail "mkfs.vfat failed" From 997d2e77c9e7feccb80c1f8f1fcf6e184ebe9489 Mon Sep 17 00:00:00 2001 From: Mike Rosseel Date: Sun, 5 Jul 2026 11:51:24 +0200 Subject: [PATCH 5/5] =?UTF-8?q?fix(migration):=20initramfs=20vs=20boot-on-?= =?UTF-8?q?ext4=20layout=20=E2=80=94=20stage=20firmware=20aside,=20verify?= =?UTF-8?q?=20per-partition,=20never=20panic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/scripts/nixos_migration_init.sh | 53 +++++++++++++++++--------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/python/scripts/nixos_migration_init.sh b/python/scripts/nixos_migration_init.sh index c0679b31..bc115150 100755 --- a/python/scripts/nixos_migration_init.sh +++ b/python/scripts/nixos_migration_init.sh @@ -111,6 +111,11 @@ fail() { exec /bin/sh } +# set -e turns ANY uncaught failure into init exiting -> kernel panic +# ("Attempted to kill init"). Route it into fail() instead: fail() ends in +# exec or reboot, so the trap never re-fires. +trap 'fail "unexpected failure near stage ${STAGE_NUM}"' EXIT + if [ -f /migration_meta ]; then . /migration_meta export MIGRATION_DISPLAY_CLASS="${DISPLAY_CLASS:-}" @@ -299,14 +304,19 @@ rm -f /tmp/migration.tar.zst show 60 "Moving rootfs" +# The tarball's top-level boot/ is the FIRMWARE partition payload; rootfs/ +# carries its own non-empty /boot (extlinux + kernels live on ext4). Stage the +# firmware payload aside first or the rootfs move collides on "boot". +mv "${MOUNT_NEW}/boot" "${MOUNT_NEW}/.fw-staging" || fail "Cannot stage firmware payload" + # Move rootfs/ contents up to partition root (same-fs rename, fast) -cd "${MOUNT_NEW}/rootfs" +cd "${MOUNT_NEW}/rootfs" || fail "rootfs missing from tarball" for item in * .[!.]* ..?*; do [ -e "$item" ] || continue - mv "$item" "${MOUNT_NEW}/" + mv "$item" "${MOUNT_NEW}/" || fail "Cannot move rootfs/${item}" done cd / -rmdir "${MOUNT_NEW}/rootfs" +rmdir "${MOUNT_NEW}/rootfs" || fail "rootfs dir not empty after move" # NetworkManager (like other security-sensitive plugin loaders) refuses to load # any plugin file not owned by root, so a /nix/store with non-root paths baked @@ -321,28 +331,33 @@ show 66 "Copying boot" mkdir -p "${MOUNT_BOOT}" mount -t vfat "${BOOT_DEV}" "${MOUNT_BOOT}" || fail "Cannot mount boot" -# Copy boot files to FAT partition -cd "${MOUNT_NEW}/boot" +# Copy the staged firmware payload to the FAT partition +cd "${MOUNT_NEW}/.fw-staging" || fail "firmware staging missing" for item in *; do [ -e "$item" ] || continue if [ -d "$item" ]; then - cp -r "$item" "${MOUNT_BOOT}/$item" + cp -r "$item" "${MOUNT_BOOT}/$item" || fail "Cannot copy ${item} to firmware partition" else - cp "$item" "${MOUNT_BOOT}/$item" + cp "$item" "${MOUNT_BOOT}/$item" || fail "Cannot copy ${item} to firmware partition" fi done cd / +rm -rf "${MOUNT_NEW}/.fw-staging" sync -# Verify critical boot files landed -if [ ! -f "${MOUNT_BOOT}/extlinux/extlinux.conf" ]; then - echo "Boot partition contents:" >&2 +# Verify each partition got what its boot chain needs: the firmware partition +# feeds the RPi firmware + U-Boot; extlinux.conf and kernels live on ext4 +# (U-Boot reads them from mmc 0:2). +if [ ! -f "${MOUNT_BOOT}/config.txt" ]; then + echo "Firmware partition contents:" >&2 ls -lR "${MOUNT_BOOT}" >&2 - fail "extlinux.conf missing from boot partition after copy" + fail "config.txt missing from firmware partition after copy" +fi +if [ ! -f "${MOUNT_NEW}/boot/extlinux/extlinux.conf" ]; then + echo "Root /boot contents:" >&2 + ls -lR "${MOUNT_NEW}/boot" >&2 + fail "extlinux.conf missing from root /boot after move" fi - -# Keep boot/ on ext4 — U-Boot reads extlinux.conf from mmc 0:2 (ext4 root) -# FAT partition only needs RPi firmware files (config.txt, u-boot, DTBs) # ------------------------------------------------------------------- # Phase 7: Migrate WiFi @@ -400,19 +415,23 @@ resize2fs "${ROOT_DEV}" 2>/dev/null || true show 92 "Syncing" sync -# Final verification: remount boot partition and confirm extlinux.conf survived +# Final verification: remount the firmware partition and confirm the RPi +# firmware config survived (extlinux.conf lives on the ext4 root, verified +# earlier). show 95 "Verifying boot" mkdir -p /mnt/bootchk mount -t vfat -o ro "${BOOT_DEV}" /mnt/bootchk || fail "Cannot remount boot for verification" -if [ ! -f /mnt/bootchk/extlinux/extlinux.conf ]; then +if [ ! -f /mnt/bootchk/config.txt ]; then ls -lR /mnt/bootchk > /dev/console 2>&1 || true umount /mnt/bootchk 2>/dev/null || true - fail "extlinux.conf missing from boot partition before reboot" + fail "config.txt missing from firmware partition before reboot" fi umount /mnt/bootchk show 100 "Complete" sleep 3 +# Success: disarm the failure trap before the deliberate reboot. +trap - EXIT echo "Rebooting into NixOS..." > /dev/console 2>/dev/null || true reboot -f