From 5629f15b19849034a4617db4cf1cf88d8596cbbe Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Tue, 13 Oct 2026 19:03:21 +0200 Subject: [PATCH 1/2] refactor(two-node): translate the Go event collector to a Python service The Go toolchain pinned in docker/Dockerfile had drifted ~15 months behind on stdlib patches, failing the daily security scan. Replace tools/event- collector with simplyblock_core/services/two_node_event_collector.py, reusing the jc_wait_events RPC client and arbitration.events.queue_event path the Python arbiter already consumes. --- docker/Dockerfile | 31 --- docs/design/two-node-arbitration.md | 12 +- docs/design/two-node-kubernetes-fencing.md | 4 +- simplyblock_core/arbitration/arbiter.py | 2 +- simplyblock_core/arbitration/events.py | 4 +- simplyblock_core/constants.py | 7 +- simplyblock_core/models/arbitration.py | 2 +- .../scripts/docker-compose-swarm.yml | 2 +- simplyblock_core/services/two_node_arbiter.py | 2 +- .../services/two_node_event_collector.py | 173 ++++++++++++++ .../test_two_node_event_collector.py | 215 ++++++++++++++++++ tests/integration/two_node_sim/AGENTS.md | 2 +- tests/integration/two_node_sim/world.py | 4 +- .../test_two_node_event_collector_idle.py | 22 ++ tools/event-collector/fakes_test.go | 107 --------- tools/event-collector/go.mod | 5 - tools/event-collector/go.sum | 2 - tools/event-collector/main.go | 50 ---- tools/event-collector/rpc.go | 133 ----------- tools/event-collector/store.go | 71 ------ tools/event-collector/store_fdb.go | 94 -------- tools/event-collector/store_file.go | 102 --------- tools/event-collector/store_nofdb.go | 11 - tools/event-collector/store_test.go | 22 -- tools/event-collector/supervisor.go | 150 ------------ tools/event-collector/supervisor_test.go | 77 ------- tools/event-collector/worker.go | 199 ---------------- tools/event-collector/worker_test.go | 154 ------------- 28 files changed, 433 insertions(+), 1226 deletions(-) create mode 100644 simplyblock_core/services/two_node_event_collector.py create mode 100644 tests/integration/test_two_node_event_collector.py create mode 100644 tests/unit/test_two_node_event_collector_idle.py delete mode 100644 tools/event-collector/fakes_test.go delete mode 100644 tools/event-collector/go.mod delete mode 100644 tools/event-collector/go.sum delete mode 100644 tools/event-collector/main.go delete mode 100644 tools/event-collector/rpc.go delete mode 100644 tools/event-collector/store.go delete mode 100644 tools/event-collector/store_fdb.go delete mode 100644 tools/event-collector/store_file.go delete mode 100644 tools/event-collector/store_nofdb.go delete mode 100644 tools/event-collector/store_test.go delete mode 100644 tools/event-collector/supervisor.go delete mode 100644 tools/event-collector/supervisor_test.go delete mode 100644 tools/event-collector/worker.go delete mode 100644 tools/event-collector/worker_test.go diff --git a/docker/Dockerfile b/docker/Dockerfile index ef43f5da42..3af0d94665 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,15 +3,6 @@ ARG UBI_VERSION=10.3 ARG FDB_VERSION=7.3.69 ARG UV_VERSION=0.13.6 -# Toolchain of the two-node HA event collector (tools/event-collector). The Go -# toolchain links its own stdlib statically into the built binary, so this is -# also what the image's scanned Go stdlib version is (grype flags it as -# `stdlib`, package type go-module) — bump it whenever a scan flags a stdlib -# CVE with no fix in 1.24.x, same as the CACHE_KEY-driven OS-package refresh -# does for dnf packages. go.mod's `go 1.24` line is an unrelated minimum -# language-version requirement, not a toolchain pin, so any newer toolchain -# here keeps building it fine. -ARG GO_VERSION=1.27.4 # The application interpreter, free-threaded ("t" suffix). Managed by uv rather # than dnf: UBI carries no free-threaded build, and tying the runtime to the @@ -201,33 +192,11 @@ RUN uv sync --locked --no-default-groups --python "${PYTHON_VERSION}t" RUN rm -rf /opt/venv/lib/python*/site-packages/kmip/demos -# The two-node HA event collector (tools/event-collector, docs/design/two-node- -# arbitration.md section 4): a Go binary linked against the FoundationDB client -# library the base stage installs, so cgo and not static. Its own stage keeps -# the compiler and the Go toolchain out of the runtime image. -FROM base AS event-collector - -ARG TARGETARCH -ARG GO_VERSION - -RUN set -eux; \ - dnf install -y gcc && dnf clean all; \ - curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" | tar -C /usr/local -xz - -WORKDIR /src -COPY tools/event-collector/go.mod tools/event-collector/go.sum ./ -RUN /usr/local/go/bin/go mod download -COPY tools/event-collector/ ./ -RUN CGO_ENABLED=1 /usr/local/go/bin/go build -tags fdb -trimpath -ldflags="-s -w" \ - -o /out/sb-event-collector . - - FROM base AS runtime COPY --from=builder /opt/python /opt/python COPY --from=builder /opt/venv /opt/venv COPY --chown=simplyblock:simplyblock . /app -COPY --from=event-collector /out/sb-event-collector /usr/local/bin/sb-event-collector # The services log through the container's gelf log driver, and a buffered # stdout loses the tail of the log when a process dies. diff --git a/docs/design/two-node-arbitration.md b/docs/design/two-node-arbitration.md index 0ad7533a9c..477b9e0b78 100644 --- a/docs/design/two-node-arbitration.md +++ b/docs/design/two-node-arbitration.md @@ -5,7 +5,7 @@ campaign c1 (no preferred node; bookkeeping, explicit-sync, fence-port, SPDK-dow rules). Design rationale, diagrams and failure analysis: [`two-node-ha-design.pdf`](two-node-ha-design.pdf) (revision 2; where it describes a preferred node, this document wins). This document is the binding contract between the work packages: -ultra (JC, distrib), the SPDK fork (lvol, blobstore), sbcli (arbiter, Go event collector, API) +ultra (JC, distrib), the SPDK fork (lvol, blobstore), sbcli (arbiter, event collector, API) and the simplyblock operator. Scope: clusters whose storage-node membership is exactly two (`apply_jc_dual_node` in @@ -204,11 +204,11 @@ Capability probe: the CP calls `rpc_get_methods`; the protocol is used only if i `jc_unfence`. Older nodes keep today's behaviour and are read with `distr_status_events_get` every 1 s. -## 4. Collector (sbcli, Go) +## 4. Collector (sbcli) -- Runs in the CP image as its own process, one goroutine per storage node of each cluster with - `two_node_arbitration` on. -- Each goroutine loops on `jc_wait_events`; on transport errors it reconnects with backoff +- Runs in the CP image as its own process (`simplyblock_core/services/two_node_event_collector.py`), + one thread per storage node of each cluster with `two_node_arbitration` on. +- Each thread loops on `jc_wait_events`; on transport errors it reconnects with backoff (100 ms, doubling, max 5 s) and re-sends the last `instance` and `after_seq`. - On `instance` change or `overflow: true`: call `jc_ha_status`, emit a synthetic `ha_resync` event carrying the full status to the arbiter, continue from the returned `seq`. @@ -757,7 +757,7 @@ Kubernetes hosts run the kernel defaults instead of these values until the drive | ultra | Hold trigger in dual-node mode; lease and epoch; `jc_grant_solo`, `jc_fence`, `jc_unfence`, `jc_lease_renew`, `jc_ha_status`, `jc_wait_events`; event fields and new statuses; resume gating after JCERR | | spdk fork | Hold on JC's signal through the frozen-IO queue (`blobstore.c:blob_execute_queued_io`); path error instead of EIO on fence; no rule removal while fenced (`lvol.c:spdk_lvs_remove_rules_poller`) | | sbcli | Arbiter service, `ClusterArbitration` model, API, verdict RPC client methods, healing flow, feature flag; RPC client methods for the new RPCs | -| sbcli | Go collector (section 4), packaged in the CP image | +| sbcli | Collector (section 4), packaged in the CP image | | operator | Taint handling, remediation status, CR fields | | test | Unit (arbiter table, epochs, lease arithmetic, long-poll sequencing); two-SPDK tests with tc/iptables; bed scenarios: storage link cut, uplink cut, both, power-off, SIGSTOP, CP restart mid-hold, latency injection; OpenShift two-node with arbiter and with fencing | diff --git a/docs/design/two-node-kubernetes-fencing.md b/docs/design/two-node-kubernetes-fencing.md index c41021bc30..932c01e26e 100644 --- a/docs/design/two-node-kubernetes-fencing.md +++ b/docs/design/two-node-kubernetes-fencing.md @@ -233,7 +233,7 @@ survivor then provably runs alone. The storage never decides; it waits for that ### 4.2 Control plane - The arbiter does not act on the cluster at all: no leases, epochs, grants, fences, unfences, - heals or leadership restoration. The Go collector does not poll its nodes (the node agent owns + heals or leadership restoration. The collector does not poll its nodes (the node agent owns the single `jc_wait_events` waiter), legacy remote-JM events are not queued for the arbiter, and no planned-loss markers are written. - `jc_set_dual_node` still carries the full arbitration config on every call (node add, restart @@ -322,7 +322,7 @@ Read-only; the agent never asks Pacemaker to fence. In order of preference: | sbcli | settings, `client_host_nqns`, `host_fences`, API (section 3.5) | both | | sbcli | host fence lifecycle in the arbiter (section 3.2) | witness | | sbcli | winner policy for partitions (section 3.3) | witness | -| sbcli | `pacemaker` gates: arbiter, Go collector, event queue, planned loss (section 4.2) | pacemaker | +| sbcli | `pacemaker` gates: arbiter, collector, event queue, planned loss (section 4.2) | pacemaker | | operator | fence agent DaemonSet, `StorageFenced`, `StorageFenceRemediation(Template)` CRDs + controller, NHC sample, release protocol | both | | operator | node agent duties of section 4.3, Pacemaker readers of section 4.5 | pacemaker | | ultra + SPDK fork | grant lifts a self-fence (section 4.4) | pacemaker | diff --git a/simplyblock_core/arbitration/arbiter.py b/simplyblock_core/arbitration/arbiter.py index 9adecd5f64..483f995056 100644 --- a/simplyblock_core/arbitration/arbiter.py +++ b/simplyblock_core/arbitration/arbiter.py @@ -252,7 +252,7 @@ def consume_events(self, cluster_id: str) -> int: legacy = item.instance == ev.LEGACY_INSTANCE if legacy and self.capable.get(item.node_id): # JC pushes a remote-JM event into both queues; on a node with - # jc_wait_events the Go collector already delivered it. + # jc_wait_events the event collector already delivered it. item.remove(self.db.kv_store) continue sig = self.signals.setdefault(item.node_id, ev.NodeSignals()) diff --git a/simplyblock_core/arbitration/events.py b/simplyblock_core/arbitration/events.py index 8a57e4ca54..c1811d15c2 100644 --- a/simplyblock_core/arbitration/events.py +++ b/simplyblock_core/arbitration/events.py @@ -128,8 +128,8 @@ def forward_legacy_jm_event(db, cluster, node_id: str, event_dict: dict) -> bool the Python distr collector to the arbiter. The distr event queue is consumed destructively by that collector - (``distr_status_events_discard_then_get``), so the Go collector must not - read it as well; this is the fallback path instead. The events carry no + (``distr_status_events_discard_then_get``), so the two-node event collector + must not read it as well; this is the fallback path instead. The events carry no sequence, so the receive time in ns orders and identifies them. Returns True when the event was a remote-JM event (handled here). """ diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 971c831f28..457d95004d 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1467,11 +1467,16 @@ def get_config_var(name, default=None): TWO_NODE_LEASE_MARGIN_MS = 500 TWO_NODE_STABLE_FOR_S = 30 TWO_NODE_WAIT_TIMEOUT_MS = 20000 +#: jc_wait_events max events per answer (contract section 3.7). +TWO_NODE_COLLECTOR_MAX_EVENTS = 256 +#: How often the collector re-reads cluster/node topology to start or stop +#: per-node collector threads. +TWO_NODE_COLLECTOR_RECONCILE_S = 5 TWO_NODE_VERDICT_HISTORY = 50 #: Arbiter decision loop period (events are folded and decided every tick). TWO_NODE_DECISION_TICK_MS = 100 #: With no cluster that has ``two_node_arbitration`` on, the arbiter (and the -#: Go event collector) re-scan the cluster list only this often instead of every +#: event collector) re-scan the cluster list only this often instead of every #: tick, so deployments without two-node clusters carry almost no FDB load. TWO_NODE_IDLE_RESCAN_S = 30 #: How often the arbiter re-reads each node's ``jc_ha_status`` and replaces its diff --git a/simplyblock_core/models/arbitration.py b/simplyblock_core/models/arbitration.py index ab3b31b8bc..810e0f3c9e 100644 --- a/simplyblock_core/models/arbitration.py +++ b/simplyblock_core/models/arbitration.py @@ -119,7 +119,7 @@ def lease_expires_at(self, node_id: str) -> int: class ArbitrationEvent(BaseModel): - """One HA event from a node, queued by the Go collector for the arbiter. + """One HA event from a node, queued by the event collector for the arbiter. Key ``{cluster_id}/{node_id}/{instance}/{seq:020d}``: a prefix read per cluster returns them in order per node, and a redelivered event (at-least- diff --git a/simplyblock_core/scripts/docker-compose-swarm.yml b/simplyblock_core/scripts/docker-compose-swarm.yml index 703b1a9892..5c96b1b58e 100644 --- a/simplyblock_core/scripts/docker-compose-swarm.yml +++ b/simplyblock_core/scripts/docker-compose-swarm.yml @@ -112,7 +112,7 @@ services: TwoNodeEventCollector: <<: *service-base image: $SIMPLYBLOCK_DOCKER_IMAGE - command: "/usr/local/bin/sb-event-collector" + command: "python3 simplyblock_core/services/two_node_event_collector.py" networks: - hostnet deploy: diff --git a/simplyblock_core/services/two_node_arbiter.py b/simplyblock_core/services/two_node_arbiter.py index 8400c8054c..c97f09f4e5 100644 --- a/simplyblock_core/services/two_node_arbiter.py +++ b/simplyblock_core/services/two_node_arbiter.py @@ -5,7 +5,7 @@ - the lease loop renews every arbitrated node's lease every ``TWO_NODE_LEASE_RENEW_MS`` (its own thread, so a slow decision never delays a renewal); -- the decision loop folds the events the Go collector queued in FDB, decides +- the decision loop folds the events the event collector queued in FDB, decides and applies verdicts every ``TWO_NODE_DECISION_TICK_MS``. Only clusters with ``two_node_arbitration`` on and exactly two members are diff --git a/simplyblock_core/services/two_node_event_collector.py b/simplyblock_core/services/two_node_event_collector.py new file mode 100644 index 0000000000..323c53f5cc --- /dev/null +++ b/simplyblock_core/services/two_node_event_collector.py @@ -0,0 +1,173 @@ +"""Two-node HA event collector (docs/design/two-node-arbitration.md, section 4). + +One thread per storage node of every cluster with ``two_node_arbitration`` on, +long-polling ``jc_wait_events`` and queuing each event for the arbiter +(``arbitration.events.queue_event``) as an ``ArbitrationEvent`` in FDB. The +connection is always opened by the control plane; nodes never call back. +""" + +import threading +import time + +from simplyblock_core import constants, db_controller, utils +from simplyblock_core.arbitration import events as arbitration_events +from simplyblock_core.arbitration.arbiter import witness_authority +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCException + +utils.init_sentry_sdk() +logger = utils.get_logger(__name__) + +db = db_controller.DBController() + +#: Backoff on a transport error: starts here, doubles, caps at BACKOFF_MAX_S. +BACKOFF_MIN_S = 0.1 +BACKOFF_MAX_S = 5.0 + +#: Synthetic event carrying a full jc_ha_status, queued on resync. +STATUS_RESYNC = "ha_resync" + + +def _resync(client, cluster_id: str, node_id: str, reported_instance: str, reason: str) -> tuple[str, int]: + """Node restart (instance change) or a dropped event (overflow): fetch the + full ``jc_ha_status`` and queue it as one ``ha_resync`` event, so the + arbiter's view is replaced rather than built back up one event at a time. + Returns the ``(instance, seq)`` to continue the long-poll from. + """ + status = client.jc_ha_status() or {} + instance = str(status.get("instance") or reported_instance) + seq = int(status.get("seq", 0)) + payload = {"status": STATUS_RESYNC, "instance": instance, "seq": seq, + "reason": reason, "ha_status": status} + arbitration_events.queue_event( + db, cluster_id, node_id, instance, seq, payload, int(time.time() * 1000)) + logger.info(f"Node {node_id} resynced ({reason}): instance={instance} seq={seq}") + return instance, seq + + +def _poll_once(client, cluster_id: str, node_id: str, instance: str, after_seq: int) -> tuple[str, int]: + """One jc_wait_events call; queues what it returns. Raises on a transport + error so the caller can back off and retry with the same (instance, + after_seq) -- at-least-once delivery, contract section 3.7.""" + res = client.jc_wait_events( + instance, after_seq, + timeout_ms=constants.TWO_NODE_WAIT_TIMEOUT_MS, + max_events=constants.TWO_NODE_COLLECTOR_MAX_EVENTS) or {} + + if res.get("superseded"): + # Another waiter took over (e.g. this collector restarted elsewhere). + return instance, after_seq + + res_instance = str(res.get("instance", "")) + overflow = bool(res.get("overflow")) + if res_instance != instance or overflow: + reason = "overflow" if overflow and res_instance == instance else "instance" + return _resync(client, cluster_id, node_id, res_instance, reason) + + events = res.get("events") or [] + received_at = int(time.time() * 1000) + for event in events: + seq = int(event.get("seq", 0)) + if seq <= after_seq: + continue + event.setdefault("instance", instance) + arbitration_events.queue_event(db, cluster_id, node_id, instance, seq, event, received_at) + after_seq = seq + + reported_seq = int(res.get("seq", 0)) + if reported_seq > after_seq and not events: + # An empty answer reports the node's current seq: nothing newer is pending. + after_seq = reported_seq + return instance, after_seq + + +def _collect(node_id: str, cluster_id: str, stop: threading.Event) -> None: + logger.info(f"Starting two-node event collector on node: {node_id}") + instance, after_seq = "", 0 + backoff = BACKOFF_MIN_S + while not stop.is_set(): + try: + snode = db.get_storage_node_by_id(node_id) + except KeyError: + logger.info(f"Node {node_id} deleted; stopping two-node event collector") + return + if snode.status == StorageNode.STATUS_REMOVED: + logger.info(f"Node {node_id} removed; stopping two-node event collector") + return + + try: + client = snode.rpc_client(retry=1) + instance, after_seq = _poll_once(client, cluster_id, node_id, instance, after_seq) + backoff = BACKOFF_MIN_S + except RPCException as e: + logger.warning(f"jc_wait_events failed on {node_id}: {e}; retry in {backoff:.1f}s") + stop.wait(backoff) + backoff = min(backoff * 2, BACKOFF_MAX_S) + + logger.info(f"Stopping two-node event collector on node: {node_id}") + + +#: node_id -> (thread, stop event) for every running collector. +_workers: dict[str, tuple[threading.Thread, threading.Event]] = {} + + +def _reconcile() -> None: + """Start a collector thread per node of every arbitrated cluster; stop any + whose node is gone, removed, or no longer in an arbitrated cluster.""" + want: dict[str, str] = {} + try: + # Mode pacemaker (docs/design/two-node-kubernetes-fencing.md, section + # 4.2): the node agent owns the single jc_wait_events waiter and this + # collector leaves the cluster alone. + clusters = [c for c in db.get_clusters() if c.two_node_arbitration and witness_authority(c)] + except Exception as e: + logger.error(f"Failed to get clusters: {e}") + return + for cluster in clusters: + cluster_id = cluster.get_id() + try: + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + except Exception as e: + logger.error(f"Failed to get nodes for cluster {cluster_id}: {e}") + continue + for snode in nodes: + if snode.status == StorageNode.STATUS_REMOVED or not snode.mgmt_ip or not snode.rpc_port: + continue + want[snode.get_id()] = cluster_id + + for node_id in [n for n in _workers if n not in want]: + thread, stop = _workers.pop(node_id) + stop.set() + thread.join(timeout=constants.TWO_NODE_WAIT_TIMEOUT_MS / 1000.0 + 15) + logger.info(f"Stopped two-node event collector for node: {node_id}") + + for node_id, cluster_id in want.items(): + existing = _workers.get(node_id) + if existing is not None and existing[0].is_alive(): + continue + stop = threading.Event() + thread = threading.Thread( + target=_collect, args=(node_id, cluster_id, stop), daemon=True, + name=f"two-node-collector-{node_id}") + thread.start() + _workers[node_id] = (thread, stop) + + +def _next_wait_s() -> float: + """``TWO_NODE_COLLECTOR_RECONCILE_S`` while any collector thread runs, + ``TWO_NODE_IDLE_RESCAN_S`` while none do -- deployments without two-node + clusters are scanned rarely instead of on every tick.""" + if _workers: + return constants.TWO_NODE_COLLECTOR_RECONCILE_S + return constants.TWO_NODE_IDLE_RESCAN_S + + +def main() -> None: + logger.info("Two-node event collector started") + while True: + _reconcile() + time.sleep(_next_wait_s()) + + +if __name__ == "__main__": + main() diff --git a/tests/integration/test_two_node_event_collector.py b/tests/integration/test_two_node_event_collector.py new file mode 100644 index 0000000000..56bd3199d2 --- /dev/null +++ b/tests/integration/test_two_node_event_collector.py @@ -0,0 +1,215 @@ +"""Two-node event collector (docs/design/two-node-arbitration.md, section 4) +against a real FoundationDB. The storage node is mocked: a plain fake RPC +client stands in for the SPDK JSON-RPC endpoint. +""" + +import threading +import uuid as _uuid + +import pytest + +from simplyblock_core import db_controller +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import two_node_event_collector as collector + + +class FakeRPC: + def __init__(self): + self.ha_status: dict = {} + self.wait_responses: list[dict] = [] + + def jc_wait_events(self, instance, after_seq, timeout_ms=20000, max_events=256): + if self.wait_responses: + return self.wait_responses.pop(0) + return {"instance": instance, "seq": after_seq, "overflow": False, "events": []} + + def jc_ha_status(self): + return self.ha_status + + +def _cluster(cid: str, arbitration: bool = True) -> Cluster: + c = Cluster() + c.uuid = cid + c.two_node_arbitration = arbitration + return c + + +def _node(nid: str, cid: str) -> StorageNode: + n = StorageNode() + n.uuid = nid + n.cluster_id = cid + n.status = StorageNode.STATUS_ONLINE + n.mgmt_ip = "127.0.0.1" + n.rpc_port = 8080 + return n + + +@pytest.fixture +def db(): + return db_controller.DBController() + + +@pytest.fixture +def ids(): + return f"cl-{_uuid.uuid4()}", f"n-{_uuid.uuid4()}" + + +def test_poll_once_queues_new_events_in_order(db, ids): + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + + rpc = FakeRPC() + rpc.wait_responses = [ + {"instance": "i1", "seq": 2, "overflow": False, + "events": [{"seq": 1, "status": "ha_solo", "jm_vuid": 3}, + {"seq": 2, "status": "ha_fenced", "jm_vuid": 3}]}, + ] + instance, after_seq = collector._poll_once(rpc, cid, nid, "i1", 0) + assert (instance, after_seq) == ("i1", 2) + events = db.get_arbitration_events(cid) + assert [e.seq for e in events] == [1, 2] + assert [e.payload["status"] for e in events] == ["ha_solo", "ha_fenced"] + + +def test_poll_once_ignores_redelivered_events(db, ids): + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + rpc = FakeRPC() + rpc.wait_responses = [{"instance": "i1", "seq": 5, "overflow": False, + "events": [{"seq": 5, "status": "ha_solo", "jm_vuid": 3}]}] + instance, after_seq = collector._poll_once(rpc, cid, nid, "i1", 5) + assert after_seq == 5 + assert db.get_arbitration_events(cid) == [] + + +def test_poll_once_empty_answer_advances_to_reported_seq(db, ids): + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + rpc = FakeRPC() + rpc.wait_responses = [{"instance": "i1", "seq": 7, "overflow": False, "events": []}] + instance, after_seq = collector._poll_once(rpc, cid, nid, "i1", 3) + assert (instance, after_seq) == ("i1", 7) + assert db.get_arbitration_events(cid) == [] + + +def test_poll_once_resyncs_on_first_call(db, ids): + """A collector thread starts with instance="" (nothing seen yet), which + never matches a node's real instance -- so the very first poll always + resyncs instead of trickling in from seq 0.""" + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + rpc = FakeRPC() + rpc.ha_status = {"instance": "i1", "seq": 3, "lvs": []} + rpc.wait_responses = [{"instance": "i1", "seq": 2, "overflow": False, "events": []}] + instance, after_seq = collector._poll_once(rpc, cid, nid, "", 0) + assert (instance, after_seq) == ("i1", 3) + assert db.get_arbitration_events(cid)[0].payload["reason"] == "instance" + + +def test_poll_once_resyncs_on_instance_change(db, ids): + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + rpc = FakeRPC() + rpc.ha_status = {"instance": "i2", "seq": 9, "lvs": []} + rpc.wait_responses = [{"instance": "i2", "seq": 1, "overflow": False, "events": []}] + instance, after_seq = collector._poll_once(rpc, cid, nid, "i1", 40) + assert (instance, after_seq) == ("i2", 9) + events = db.get_arbitration_events(cid) + assert len(events) == 1 + assert events[0].payload["status"] == collector.STATUS_RESYNC + assert events[0].payload["reason"] == "instance" + + +def test_poll_once_resyncs_on_overflow(db, ids): + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + rpc = FakeRPC() + rpc.ha_status = {"instance": "i1", "seq": 100, "lvs": []} + rpc.wait_responses = [{"instance": "i1", "seq": 50, "overflow": True, "events": []}] + instance, after_seq = collector._poll_once(rpc, cid, nid, "i1", 10) + assert (instance, after_seq) == ("i1", 100) + events = db.get_arbitration_events(cid) + assert events[0].payload["reason"] == "overflow" + + +def test_poll_once_superseded_is_a_noop(db, ids): + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + rpc = FakeRPC() + rpc.wait_responses = [{"superseded": True}] + instance, after_seq = collector._poll_once(rpc, cid, nid, "i1", 10) + assert (instance, after_seq) == ("i1", 10) + assert db.get_arbitration_events(cid) == [] + + +def test_reconcile_starts_and_stops_collector_threads(db, ids, monkeypatch): + cid, nid = ids + cluster = _cluster(cid) + cluster.write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + + started = threading.Event() + + def fake_collect(node_id, cluster_id, stop): + started.set() + stop.wait() + + monkeypatch.setattr(collector, "_collect", fake_collect) + collector._workers.clear() + try: + collector._reconcile() + assert started.wait(timeout=2) + assert nid in collector._workers + + cluster.two_node_arbitration = False + cluster.write_to_db(db.kv_store) + collector._reconcile() + assert nid not in collector._workers + finally: + for thread, stop in collector._workers.values(): + stop.set() + thread.join(timeout=2) + collector._workers.clear() + + +def test_reconcile_skips_a_pacemaker_authority_cluster(db, ids, monkeypatch): + """Mode pacemaker (docs/design/two-node-kubernetes-fencing.md, section + 4.2): the node agent owns the single jc_wait_events waiter, so the + collector must leave the cluster alone.""" + cid, nid = ids + cluster = _cluster(cid) + cluster.two_node_fencing_authority = "pacemaker" + cluster.write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + + monkeypatch.setattr(collector, "_collect", lambda *a: None) + collector._workers.clear() + try: + collector._reconcile() + assert nid not in collector._workers + finally: + collector._workers.clear() + + +def test_reconcile_skips_a_removed_node(db, ids, monkeypatch): + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + node = _node(nid, cid) + node.status = StorageNode.STATUS_REMOVED + node.write_to_db(db.kv_store) + + monkeypatch.setattr(collector, "_collect", lambda *a: None) + collector._workers.clear() + try: + collector._reconcile() + assert nid not in collector._workers + finally: + collector._workers.clear() diff --git a/tests/integration/two_node_sim/AGENTS.md b/tests/integration/two_node_sim/AGENTS.md index 975010ca06..0de647f280 100644 --- a/tests/integration/two_node_sim/AGENTS.md +++ b/tests/integration/two_node_sim/AGENTS.md @@ -19,7 +19,7 @@ and one regression scenario per finding, in CI: | the arbiter's default helpers: leadership restore, device check, rebuild-task bookkeeping, the leaderless reconcile and its probe | the network (storage link, management link, client link, lossy link) | | every `RPCClient` wrapper — only `RPCClient._request` is routed to the simulated node | the client's NVMe multipath IO (KATO 4 s, fast_io_fail 8 s) | | the guards the flows consult: `_single_peer_jm_is_complete`, `_two_node_fence_holds_ports`, `_may_unblock_peer_port`, `_jc_compression_status_with_retry`, `_failover_primary_ana`, `_failback_primary_ana`, `_set_lvol_ana_on_node` | the outer shell of the flows the arbiter does not own: monitor (offline, auto-restart, stale-port remediation), restart runner, port-allow runner, migration runner, shutdown flow. Each keeps the real guard calls and models only the RPC-heavy work | -| `tasks_controller` task records, `set_node_status`, all FDB reads and writes (real FoundationDB, as the integration tier requires) | the Go event collector (same contract: in order per node, `ha_resync` on a new instance) | +| `tasks_controller` task records, `set_node_status`, all FDB reads and writes (real FoundationDB, as the integration tier requires) | the event collector (same contract: in order per node, `ha_resync` on a new instance) | Time is virtual (`vclock.py`): `time.time`/`monotonic`/`sleep` and the arbiter's clock read one clock; nothing sleeps or waits on the wall clock. A diff --git a/tests/integration/two_node_sim/world.py b/tests/integration/two_node_sim/world.py index bfe13edf5f..d11c27a910 100644 --- a/tests/integration/two_node_sim/world.py +++ b/tests/integration/two_node_sim/world.py @@ -12,7 +12,7 @@ all FDB reads and writes. What is modelled: the nodes (``simnode``), the network, the client's -multipath IO, the Go event collector, and the outer shell of the flows the +multipath IO, the event collector, and the outer shell of the flows the arbiter does not own -- the monitor's status/restart trigger, the restart runner, the migration runner, the port-allow runner and the monitor's stale-port remediation. Each modelled flow keeps the real decision points @@ -520,7 +520,7 @@ def lease_tick(self): interval = constants.TWO_NODE_LEASE_RENEW_MS / 1000.0 self.vt.after(max(interval, ctx.elapsed), self.lease_tick) - # ---- the Go collector ---------------------------------------------------------------------- + # ---- the event collector ------------------------------------------------------------------- def collector_tick(self): from simplyblock_core.arbitration import events as ev diff --git a/tests/unit/test_two_node_event_collector_idle.py b/tests/unit/test_two_node_event_collector_idle.py new file mode 100644 index 0000000000..d11efa1dda --- /dev/null +++ b/tests/unit/test_two_node_event_collector_idle.py @@ -0,0 +1,22 @@ +"""Two-node event collector: idle back-off without arbitrated clusters +(docs/design/two-node-arbitration.md, section 4). Pure logic, no DB. +""" + +import threading + +from simplyblock_core import constants +from simplyblock_core.services import two_node_event_collector as collector + + +def test_next_wait_is_idle_with_no_workers(): + collector._workers.clear() + assert collector._next_wait_s() == constants.TWO_NODE_IDLE_RESCAN_S + + +def test_next_wait_is_the_reconcile_interval_with_a_worker(): + collector._workers.clear() + collector._workers["n1"] = (threading.Thread(), threading.Event()) + try: + assert collector._next_wait_s() == constants.TWO_NODE_COLLECTOR_RECONCILE_S + finally: + collector._workers.clear() diff --git a/tools/event-collector/fakes_test.go b/tools/event-collector/fakes_test.go deleted file mode 100644 index db022811fc..0000000000 --- a/tools/event-collector/fakes_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "sync" - "time" -) - -// memStore is an in-memory Store keyed like FDB. -type memStore struct { - mu sync.Mutex - clusters []Cluster - nodes map[string][]Node - events map[string]Event - putErr error -} - -func newMemStore() *memStore { - return &memStore{nodes: map[string][]Node{}, events: map[string]Event{}} -} - -func (m *memStore) ArbitratedClusters() ([]Cluster, error) { - m.mu.Lock() - defer m.mu.Unlock() - return append([]Cluster(nil), m.clusters...), nil -} - -func (m *memStore) Nodes(id string) ([]Node, error) { - m.mu.Lock() - defer m.mu.Unlock() - return append([]Node(nil), m.nodes[id]...), nil -} - -func (m *memStore) PutEvent(e Event) error { - m.mu.Lock() - defer m.mu.Unlock() - if m.putErr != nil { - return m.putErr - } - m.events[e.Key()] = e - return nil -} - -func (m *memStore) count() int { - m.mu.Lock() - defer m.mu.Unlock() - return len(m.events) -} - -// scripted is a Caller answering from a queue per method and recording the calls. -// With nothing queued it blocks like a long-poll until the context ends. -type scripted struct { - mu sync.Mutex - answers map[string][]any // value or error - calls []call -} - -type call struct { - method string - params any -} - -func (s *scripted) push(method string, v any) { - s.mu.Lock() - defer s.mu.Unlock() - if s.answers == nil { - s.answers = map[string][]any{} - } - s.answers[method] = append(s.answers[method], v) -} - -func (s *scripted) Call(ctx context.Context, method string, params any, out any) error { - s.mu.Lock() - s.calls = append(s.calls, call{method, params}) - q := s.answers[method] - if len(q) == 0 { - s.mu.Unlock() - <-ctx.Done() - return ctx.Err() - } - v := q[0] - s.answers[method] = q[1:] - s.mu.Unlock() - if err, ok := v.(error); ok { - return err - } - raw, _ := json.Marshal(v) - return json.Unmarshal(raw, out) -} - -func (s *scripted) waitArgs() []waitArgs { - s.mu.Lock() - defer s.mu.Unlock() - var out []waitArgs - for _, c := range s.calls { - if a, ok := c.params.(waitArgs); ok { - out = append(out, a) - } - } - return out -} - -var errDown = errors.New("connection refused") - -func fixedNow() time.Time { return time.UnixMilli(1_700_000_000_000) } diff --git a/tools/event-collector/go.mod b/tools/event-collector/go.mod deleted file mode 100644 index 10593afb4a..0000000000 --- a/tools/event-collector/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/simplyblock/sbcli/tools/event-collector - -go 1.24 - -require github.com/apple/foundationdb/bindings/go v0.0.0-20250702211439-37fcf1c8ce08 diff --git a/tools/event-collector/go.sum b/tools/event-collector/go.sum deleted file mode 100644 index 9583ad8c21..0000000000 --- a/tools/event-collector/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/apple/foundationdb/bindings/go v0.0.0-20250702211439-37fcf1c8ce08 h1:FBu5q43+8pMQO5IgC+anjjz+3djUWg2jfCp7rz+IyWM= -github.com/apple/foundationdb/bindings/go v0.0.0-20250702211439-37fcf1c8ce08/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U= diff --git a/tools/event-collector/main.go b/tools/event-collector/main.go deleted file mode 100644 index 2bad736c22..0000000000 --- a/tools/event-collector/main.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "context" - "log/slog" - "os" - "os/signal" - "strings" - "syscall" - "time" -) - -func main() { - level := slog.LevelInfo - switch strings.ToUpper(os.Getenv("SIMPLYBLOCK_LOG_LEVEL")) { - case "DEBUG": - level = slog.LevelDebug - case "WARNING", "WARN": - level = slog.LevelWarn - case "ERROR": - level = slog.LevelError - } - log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: level})) - slog.SetDefault(log) - - clusterFile := os.Getenv("FDB_CLUSTER_FILE") - if clusterFile == "" { - clusterFile = "/etc/foundationdb/fdb.cluster" - } - store, isFile, err := fileStoreFromEnv() - if !isFile { - store, err = newStore(clusterFile) - } - if err != nil { - log.Error("cannot open the store", "err", err) - os.Exit(1) - } - tlsCfg := TLSConfigFromEnv() - client, err := tlsCfg.HTTPClient() - if err != nil { - log.Error("cannot set up TLS", "err", err) - os.Exit(1) - } - - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - log.Info("two-node event collector started", "tls", tlsCfg.Mode) - (&Supervisor{Store: store, TLS: tlsCfg, Client: client, Interval: 5 * time.Second, Log: log}).Run(ctx) - log.Info("two-node event collector stopped") -} diff --git a/tools/event-collector/rpc.go b/tools/event-collector/rpc.go deleted file mode 100644 index 7ef0480c9e..0000000000 --- a/tools/event-collector/rpc.go +++ /dev/null @@ -1,133 +0,0 @@ -package main - -import ( - "bytes" - "context" - "crypto/tls" - "crypto/x509" - "encoding/json" - "fmt" - "net/http" - "os" - "strings" - "time" -) - -// RPCError is a JSON-RPC error answer from the node. -type RPCError struct { - Code int `json:"code"` - Message string `json:"message"` -} - -func (e *RPCError) Error() string { return fmt.Sprintf("rpc error %d: %s", e.Code, e.Message) } - -// Caller sends one JSON-RPC request to a node. -type Caller interface { - Call(ctx context.Context, method string, params any, out any) error -} - -// HTTPCaller talks to the node's SPDK HTTP proxy the way simplyblock_core/rpc_client.py does: -// POST {"id","method","params"} to {scheme}://{mgmt_ip}:{rpc_port}/ with basic auth. -type HTTPCaller struct { - URL string - Username string - Password string - Client *http.Client -} - -func (c *HTTPCaller) Call(ctx context.Context, method string, params any, out any) error { - body := map[string]any{"id": 1, "method": method} - if params != nil { - body["params"] = params - } - raw, err := json.Marshal(body) - if err != nil { - return err - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL, bytes.NewReader(raw)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - req.SetBasicAuth(c.Username, c.Password) - resp, err := c.Client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("%s: http %d", method, resp.StatusCode) - } - var answer struct { - Result json.RawMessage `json:"result"` - Error *RPCError `json:"error"` - } - if err := json.NewDecoder(resp.Body).Decode(&answer); err != nil { - return fmt.Errorf("%s: %w", method, err) - } - if answer.Error != nil { - return answer.Error - } - if out == nil { - return nil - } - return json.Unmarshal(answer.Result, out) -} - -// TLSConfig mirrors simplyblock_core.settings: SB_TLS_CONNECT is disabled, anonymous or -// authenticated; the CA, certificate and key come from SB_TLS_CERTIFICATE_AUTHORITY, -// SB_TLS_CERTIFICATE and SB_TLS_KEY. -type TLSConfig struct { - Mode, CA, Cert, Key string -} - -// TLSConfigFromEnv reads the SB_TLS_* variables with the Python defaults. -func TLSConfigFromEnv() TLSConfig { - get := func(k, def string) string { - if v := os.Getenv(k); v != "" { - return v - } - return def - } - return TLSConfig{ - Mode: strings.ToLower(get("SB_TLS_CONNECT", "disabled")), - CA: get("SB_TLS_CERTIFICATE_AUTHORITY", "/etc/simplyblock/tls/ca.crt"), - Cert: get("SB_TLS_CERTIFICATE", "/etc/simplyblock/tls/tls.crt"), - Key: get("SB_TLS_KEY", "/etc/simplyblock/tls/tls.key"), - } -} - -// Scheme is the URL scheme for the mode. -func (t TLSConfig) Scheme() string { - if t.Mode == "" || t.Mode == "disabled" { - return "http" - } - return "https" -} - -// HTTPClient builds the client for the mode. The overall timeout must exceed the long-poll -// timeout, so it is set per request through the context instead. -func (t TLSConfig) HTTPClient() (*http.Client, error) { - tr := &http.Transport{MaxIdleConnsPerHost: 2, IdleConnTimeout: 90 * time.Second} - if t.Scheme() == "https" { - cfg := &tls.Config{MinVersion: tls.VersionTLS12} - pem, err := os.ReadFile(t.CA) - if err != nil { - return nil, fmt.Errorf("tls ca: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(pem) { - return nil, fmt.Errorf("tls ca: no certificate in %s", t.CA) - } - cfg.RootCAs = pool - if t.Mode == "authenticated" { - cert, err := tls.LoadX509KeyPair(t.Cert, t.Key) - if err != nil { - return nil, fmt.Errorf("tls client certificate: %w", err) - } - cfg.Certificates = []tls.Certificate{cert} - } - tr.TLSClientConfig = cfg - } - return &http.Client{Transport: tr}, nil -} diff --git a/tools/event-collector/store.go b/tools/event-collector/store.go deleted file mode 100644 index 2318bdde45..0000000000 --- a/tools/event-collector/store.go +++ /dev/null @@ -1,71 +0,0 @@ -// Package main is the two-node HA event collector (docs/design/two-node-arbitration.md, section 4). -// -// One goroutine per storage node of every cluster with two_node_arbitration on holds a -// jc_wait_events long-poll against the node's SPDK JSON-RPC endpoint and queues each event -// as an ArbitrationEvent record in FoundationDB, where the Python arbiter consumes it. -// The connection is always opened by the control plane; nodes never call back. -package main - -import "fmt" - -// Cluster is the part of a simplyblock Cluster record the collector needs. -type Cluster struct { - UUID string `json:"uuid"` - TwoNodeArbitration bool `json:"two_node_arbitration"` - // FencingAuthority is "witness" (default, also when empty) or "pacemaker". In mode - // pacemaker the node agent owns the single jc_wait_events waiter and no arbiter consumes - // events (docs/design/two-node-kubernetes-fencing.md, section 4.2). - FencingAuthority string `json:"two_node_fencing_authority"` -} - -// Arbitrated reports whether the collector serves the cluster's nodes. -func (c Cluster) Arbitrated() bool { - return c.TwoNodeArbitration && c.FencingAuthority != "pacemaker" -} - -// Node is the part of a StorageNode record the collector needs. -type Node struct { - UUID string `json:"uuid"` - ClusterID string `json:"cluster_id"` - Status string `json:"status"` - MgmtIP string `json:"mgmt_ip"` - RPCPort int `json:"rpc_port"` - RPCUsername string `json:"rpc_username"` - RPCPassword string `json:"rpc_password"` -} - -// Event is one queued ArbitrationEvent, serialised like the Python model -// (simplyblock_core/models/arbitration.py). -type Event struct { - ClusterID string `json:"cluster_id"` - NodeID string `json:"node_id"` - Instance string `json:"instance"` - Seq int64 `json:"seq"` - ReceivedAt int64 `json:"received_at"` - Payload map[string]any `json:"payload"` -} - -// Key is the record's FDB key: object/ArbitrationEvent/{cluster}/{node}/{instance}/{seq:020d}. -// A redelivered event overwrites its own key, so the queue never holds duplicates. -func (e Event) Key() string { - return fmt.Sprintf("object/ArbitrationEvent/%s/%s/%s/%020d", e.ClusterID, e.NodeID, e.Instance, e.Seq) -} - -// Record is the JSON value written for the event, matching BaseModel.to_dict(). -func (e Event) Record() map[string]any { - return map[string]any{ - "id": "", "uuid": "", "name": "ArbitrationEvent", "status": "", "deleted": false, - "updated_at": "", "create_dt": "", "remove_dt": "", "object_type": "object", - "cluster_id": e.ClusterID, "node_id": e.NodeID, "instance": e.Instance, - "seq": e.Seq, "received_at": e.ReceivedAt, "payload": e.Payload, - } -} - -// Store reads the cluster topology and queues events. -type Store interface { - ArbitratedClusters() ([]Cluster, error) - Nodes(clusterID string) ([]Node, error) - PutEvent(Event) error -} - -const nodeStatusRemoved = "removed" diff --git a/tools/event-collector/store_fdb.go b/tools/event-collector/store_fdb.go deleted file mode 100644 index cecafd01d1..0000000000 --- a/tools/event-collector/store_fdb.go +++ /dev/null @@ -1,94 +0,0 @@ -//go:build fdb - -package main - -import ( - "encoding/json" - "fmt" - - "github.com/apple/foundationdb/bindings/go/src/fdb" -) - -// fdbAPIVersion matches the Python control plane (constants.KVD_DB_VERSION). -const fdbAPIVersion = 730 - -// fdbStore reads and writes the same raw keys as simplyblock_core.models.base_model: -// object//, JSON values. -type fdbStore struct { - db fdb.Database -} - -func newStore(clusterFile string) (Store, error) { - if err := fdb.APIVersion(fdbAPIVersion); err != nil { - return nil, fmt.Errorf("fdb api version: %w", err) - } - db, err := fdb.OpenDatabase(clusterFile) - if err != nil { - return nil, fmt.Errorf("fdb open %s: %w", clusterFile, err) - } - return &fdbStore{db: db}, nil -} - -func (s *fdbStore) scan(prefix string) ([][]byte, error) { - kr, err := fdb.PrefixRange([]byte(prefix)) - if err != nil { - return nil, err - } - out, err := s.db.ReadTransact(func(rt fdb.ReadTransaction) (any, error) { - kvs, err := rt.GetRange(kr, fdb.RangeOptions{}).GetSliceWithError() - if err != nil { - return nil, err - } - vals := make([][]byte, 0, len(kvs)) - for _, kv := range kvs { - vals = append(vals, kv.Value) - } - return vals, nil - }) - if err != nil { - return nil, err - } - return out.([][]byte), nil -} - -func (s *fdbStore) ArbitratedClusters() ([]Cluster, error) { - vals, err := s.scan("object/Cluster/") - if err != nil { - return nil, err - } - var out []Cluster - for _, v := range vals { - var c Cluster - if json.Unmarshal(v, &c) == nil && c.Arbitrated() && c.UUID != "" { - out = append(out, c) - } - } - return out, nil -} - -func (s *fdbStore) Nodes(clusterID string) ([]Node, error) { - vals, err := s.scan("object/StorageNode/") - if err != nil { - return nil, err - } - var out []Node - for _, v := range vals { - var n Node - if json.Unmarshal(v, &n) == nil && n.ClusterID == clusterID { - out = append(out, n) - } - } - return out, nil -} - -func (s *fdbStore) PutEvent(e Event) error { - raw, err := json.Marshal(e.Record()) - if err != nil { - return err - } - _, err = s.db.Transact(func(tr fdb.Transaction) (any, error) { - tr.Set(fdb.Key(e.Key()), raw) - return nil, nil - }) - return err -} diff --git a/tools/event-collector/store_file.go b/tools/event-collector/store_file.go deleted file mode 100644 index cd973396e0..0000000000 --- a/tools/event-collector/store_file.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "strings" - "sync" - "time" -) - -// FileStore is a test sink: the topology comes from a JSON file and every event is -// appended as one JSON line, so the collector runs without the control plane or -// FoundationDB (two-node integration tests). Selected with -// SB_COLLECTOR_STORE=file::. -// -// topology.json: {"clusters":[{"uuid":"c1","two_node_arbitration":true}], -// "nodes":[{"uuid":"n1","cluster_id":"c1","status":"online","mgmt_ip":"10.0.0.1", -// "rpc_port":8080,"rpc_username":"u","rpc_password":"p"}]} -// The topology file is re-read on every call, so a test can add, remove or move nodes. -type FileStore struct { - TopologyPath string - EventsPath string - mu sync.Mutex -} - -type fileTopology struct { - Clusters []Cluster `json:"clusters"` - Nodes []Node `json:"nodes"` -} - -func (f *FileStore) topology() (fileTopology, error) { - var t fileTopology - b, err := os.ReadFile(f.TopologyPath) - if err != nil { - return t, err - } - err = json.Unmarshal(b, &t) - return t, err -} - -// ArbitratedClusters returns the clusters with two_node_arbitration on. -func (f *FileStore) ArbitratedClusters() ([]Cluster, error) { - t, err := f.topology() - if err != nil { - return nil, err - } - var out []Cluster - for _, c := range t.Clusters { - if c.Arbitrated() { - out = append(out, c) - } - } - return out, nil -} - -// Nodes returns the nodes of one cluster. -func (f *FileStore) Nodes(clusterID string) ([]Node, error) { - t, err := f.topology() - if err != nil { - return nil, err - } - var out []Node - for _, n := range t.Nodes { - if n.ClusterID == clusterID { - out = append(out, n) - } - } - return out, nil -} - -// PutEvent appends the event with the time it was written, one JSON object per line. -func (f *FileStore) PutEvent(e Event) error { - line, err := json.Marshal(map[string]any{ - "key": e.Key(), "record": e.Record(), "written_at_ms": time.Now().UnixMilli(), - }) - if err != nil { - return err - } - f.mu.Lock() - defer f.mu.Unlock() - fh, err := os.OpenFile(f.EventsPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) - if err != nil { - return err - } - defer fh.Close() - _, err = fh.Write(append(line, '\n')) - return err -} - -// fileStoreFromEnv returns the file store when SB_COLLECTOR_STORE selects it. -func fileStoreFromEnv() (Store, bool, error) { - v := os.Getenv("SB_COLLECTOR_STORE") - if !strings.HasPrefix(v, "file:") { - return nil, false, nil - } - parts := strings.SplitN(strings.TrimPrefix(v, "file:"), ":", 2) - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return nil, true, fmt.Errorf("SB_COLLECTOR_STORE=file::, got %q", v) - } - return &FileStore{TopologyPath: parts[0], EventsPath: parts[1]}, true, nil -} diff --git a/tools/event-collector/store_nofdb.go b/tools/event-collector/store_nofdb.go deleted file mode 100644 index ef1cd7d0cd..0000000000 --- a/tools/event-collector/store_nofdb.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build !fdb - -package main - -import "errors" - -// newStore without the fdb build tag: the collector needs the FoundationDB binding (cgo), -// built into the image with `go build -tags fdb` (docker/Dockerfile). -func newStore(string) (Store, error) { - return nil, errors.New("built without FoundationDB support (build with -tags fdb)") -} diff --git a/tools/event-collector/store_test.go b/tools/event-collector/store_test.go deleted file mode 100644 index 6347175ea0..0000000000 --- a/tools/event-collector/store_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import "testing" - -// Mode pacemaker (docs/design/two-node-kubernetes-fencing.md, section 4.2): the node agent -// owns the single jc_wait_events waiter, so the collector must leave the cluster alone. -func TestClusterArbitratedExcludesPacemakerAuthority(t *testing.T) { - cases := []struct { - c Cluster - want bool - }{ - {Cluster{UUID: "a", TwoNodeArbitration: true}, true}, - {Cluster{UUID: "b", TwoNodeArbitration: true, FencingAuthority: "witness"}, true}, - {Cluster{UUID: "c", TwoNodeArbitration: true, FencingAuthority: "pacemaker"}, false}, - {Cluster{UUID: "d", TwoNodeArbitration: false}, false}, - } - for _, tc := range cases { - if got := tc.c.Arbitrated(); got != tc.want { - t.Errorf("%s: Arbitrated() = %v, want %v", tc.c.UUID, got, tc.want) - } - } -} diff --git a/tools/event-collector/supervisor.go b/tools/event-collector/supervisor.go deleted file mode 100644 index 58835d1a70..0000000000 --- a/tools/event-collector/supervisor.go +++ /dev/null @@ -1,150 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log/slog" - "net/http" - "sync" - "time" -) - -// Supervisor keeps one Worker per storage node of every arbitrated cluster, starting and -// stopping them as clusters enable or disable arbitration and nodes join, leave or move. -type Supervisor struct { - Store Store - TLS TLSConfig - Client *http.Client - Interval time.Duration - // IdleInterval is the period while no cluster has arbitration on (no - // workers): deployments without two-node clusters are scanned rarely. - IdleInterval time.Duration - Log *slog.Logger - // NewCaller is replaceable in tests. - NewCaller func(Node) Caller - - mu sync.Mutex - running map[string]*running -} - -type running struct { - fingerprint string - cancel context.CancelFunc - done chan struct{} -} - -func (s *Supervisor) caller(n Node) Caller { - if s.NewCaller != nil { - return s.NewCaller(n) - } - return &HTTPCaller{ - URL: fmt.Sprintf("%s://%s:%d/", s.TLS.Scheme(), n.MgmtIP, n.RPCPort), - Username: n.RPCUsername, Password: n.RPCPassword, Client: s.Client, - } -} - -// fingerprint changes when the node must be reached differently, which restarts its worker. -func fingerprint(n Node) string { - return fmt.Sprintf("%s|%s|%d|%s|%s", n.ClusterID, n.MgmtIP, n.RPCPort, n.RPCUsername, n.RPCPassword) -} - -// Reconcile brings the set of workers in line with the store once. -func (s *Supervisor) Reconcile(ctx context.Context) error { - clusters, err := s.Store.ArbitratedClusters() - if err != nil { - return err - } - want := map[string]Node{} - for _, c := range clusters { - nodes, err := s.Store.Nodes(c.UUID) - if err != nil { - return err - } - for _, n := range nodes { - if n.Status == nodeStatusRemoved || n.MgmtIP == "" || n.RPCPort <= 0 { - continue - } - want[n.UUID] = n - } - } - - s.mu.Lock() - defer s.mu.Unlock() - if s.running == nil { - s.running = map[string]*running{} - } - for id, r := range s.running { - n, ok := want[id] - if !ok || fingerprint(n) != r.fingerprint { - r.cancel() - <-r.done - delete(s.running, id) - s.log().Info("worker stopped", "node", id) - } - } - for id, n := range want { - if _, ok := s.running[id]; ok { - continue - } - wctx, cancel := context.WithCancel(ctx) - r := &running{fingerprint: fingerprint(n), cancel: cancel, done: make(chan struct{})} - w := &Worker{ClusterID: n.ClusterID, NodeID: n.UUID, RPC: s.caller(n), Store: s.Store, Log: s.log()} - go func() { - defer close(r.done) - w.Run(wctx) - }() - s.running[id] = r - s.log().Info("worker started", "cluster", n.ClusterID, "node", id) - } - return nil -} - -// Workers is the number of running workers. -func (s *Supervisor) Workers() int { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.running) -} - -// nextWait is Interval while workers run, IdleInterval while none do. -func (s *Supervisor) nextWait() time.Duration { - interval := s.Interval - if interval == 0 { - interval = 5 * time.Second - } - idle := s.IdleInterval - if idle == 0 { - idle = 30 * time.Second - } - if s.Workers() == 0 && idle > interval { - return idle - } - return interval -} - -// Run reconciles every Interval (IdleInterval with no arbitrated cluster) until ctx -// ends, then stops all workers. -func (s *Supervisor) Run(ctx context.Context) { - for { - if err := s.Reconcile(ctx); err != nil { - s.log().Warn("reconcile failed", "err", err) - } - if sleepCtx(ctx, s.nextWait()) != nil { - break - } - } - s.mu.Lock() - defer s.mu.Unlock() - for id, r := range s.running { - r.cancel() - <-r.done - delete(s.running, id) - } -} - -func (s *Supervisor) log() *slog.Logger { - if s.Log != nil { - return s.Log - } - return slog.Default() -} diff --git a/tools/event-collector/supervisor_test.go b/tools/event-collector/supervisor_test.go deleted file mode 100644 index 7d16f0f126..0000000000 --- a/tools/event-collector/supervisor_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "context" - "testing" - "time" -) - -func TestSupervisorStartsStopsAndRestartsWorkers(t *testing.T) { - st := newMemStore() - st.clusters = []Cluster{{UUID: "cl", TwoNodeArbitration: true}} - st.nodes["cl"] = []Node{ - {UUID: "a", ClusterID: "cl", MgmtIP: "10.0.0.1", RPCPort: 8080}, - {UUID: "b", ClusterID: "cl", MgmtIP: "10.0.0.2", RPCPort: 8080}, - {UUID: "gone", ClusterID: "cl", MgmtIP: "10.0.0.3", RPCPort: 8080, Status: nodeStatusRemoved}, - } - callers := map[string]int{} - s := &Supervisor{Store: st, NewCaller: func(n Node) Caller { callers[n.UUID]++; return &scripted{} }} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if err := s.Reconcile(ctx); err != nil || s.Workers() != 2 { - t.Fatalf("workers %d err %v", s.Workers(), err) - } - // unchanged: no restart - _ = s.Reconcile(ctx) - if callers["a"] != 1 { - t.Fatalf("a restarted without a change: %v", callers) - } - // an address change restarts the worker - st.nodes["cl"][0].MgmtIP = "10.0.0.9" - _ = s.Reconcile(ctx) - if callers["a"] != 2 || s.Workers() != 2 { - t.Fatalf("callers %v workers %d", callers, s.Workers()) - } - // disabling arbitration stops every worker - st.clusters = nil - done := make(chan struct{}) - go func() { _ = s.Reconcile(ctx); close(done) }() - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("stopping workers hung") - } - if s.Workers() != 0 { - t.Fatalf("workers %d", s.Workers()) - } -} - -// Without an arbitrated cluster the supervisor scans rarely; with workers it -// returns to Interval. -func TestSupervisorWaitsLongerWhileIdle(t *testing.T) { - st := newMemStore() - s := &Supervisor{Store: st, Interval: 5 * time.Second, NewCaller: func(Node) Caller { return &scripted{} }} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - if err := s.Reconcile(ctx); err != nil { - t.Fatal(err) - } - if got := s.nextWait(); got != 30*time.Second { - t.Fatalf("idle wait %v, want 30s", got) - } - st.clusters = []Cluster{{UUID: "cl", TwoNodeArbitration: true}} - st.nodes["cl"] = []Node{{UUID: "a", ClusterID: "cl", MgmtIP: "10.0.0.1", RPCPort: 8080}} - if err := s.Reconcile(ctx); err != nil { - t.Fatal(err) - } - if got := s.nextWait(); got != 5*time.Second { - t.Fatalf("active wait %v, want 5s", got) - } - st.clusters = nil - _ = s.Reconcile(ctx) - if got := s.nextWait(); got != 30*time.Second { - t.Fatalf("idle again: wait %v, want 30s", got) - } -} diff --git a/tools/event-collector/worker.go b/tools/event-collector/worker.go deleted file mode 100644 index 7ce400a7f4..0000000000 --- a/tools/event-collector/worker.go +++ /dev/null @@ -1,199 +0,0 @@ -package main - -import ( - "context" - "errors" - "log/slog" - "strconv" - "time" -) - -// Defaults from the contract (sections 3.7 and 4). -const ( - defaultWaitTimeout = 20 * time.Second - defaultMaxEvents = 256 - backoffMin = 100 * time.Millisecond - backoffMax = 5 * time.Second - // Slack on top of the long-poll timeout before the HTTP request gives up. - httpSlack = 10 * time.Second -) - -// statusResync is the synthetic event carrying a full jc_ha_status (arbitration/events.py). -const statusResync = "ha_resync" - -type waitArgs struct { - Instance string `json:"instance"` - AfterSeq int64 `json:"after_seq"` - TimeoutMS int64 `json:"timeout_ms"` - Max int `json:"max"` -} - -type waitResult struct { - Instance string `json:"instance"` - Seq int64 `json:"seq"` - Overflow bool `json:"overflow"` - Superseded bool `json:"superseded"` - Events []map[string]any `json:"events"` -} - -// Worker holds the long-poll for one node. -type Worker struct { - ClusterID string - NodeID string - RPC Caller - Store Store - Log *slog.Logger - Now func() time.Time - Sleep func(context.Context, time.Duration) error - Timeout time.Duration - Max int - - instance string - afterSeq int64 -} - -func (w *Worker) defaults() { - if w.Now == nil { - w.Now = time.Now - } - if w.Sleep == nil { - w.Sleep = sleepCtx - } - if w.Timeout == 0 { - w.Timeout = defaultWaitTimeout - } - if w.Max == 0 { - w.Max = defaultMaxEvents - } - if w.Log == nil { - w.Log = slog.Default() - } -} - -func sleepCtx(ctx context.Context, d time.Duration) error { - t := time.NewTimer(d) - defer t.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-t.C: - return nil - } -} - -// Run loops until ctx ends. Transport errors back off from 100 ms, doubling, up to 5 s, and -// the next call re-sends the last instance and after_seq (at-least-once delivery). -func (w *Worker) Run(ctx context.Context) { - w.defaults() - backoff := backoffMin - for ctx.Err() == nil { - if err := w.Once(ctx); err != nil { - if ctx.Err() != nil { - return - } - w.Log.Warn("jc_wait_events failed", "node", w.NodeID, "err", err, "retry_in", backoff) - if w.Sleep(ctx, backoff) != nil { - return - } - backoff = min(backoff*2, backoffMax) - continue - } - backoff = backoffMin - } -} - -// Once performs one jc_wait_events call and queues what it returns. The acknowledgement -// (after_seq) advances only after every event is stored, so a failed write is redelivered. -func (w *Worker) Once(ctx context.Context) error { - w.defaults() - callCtx, cancel := context.WithTimeout(ctx, w.Timeout+httpSlack) - defer cancel() - var res waitResult - err := w.RPC.Call(callCtx, "jc_wait_events", - waitArgs{Instance: w.instance, AfterSeq: w.afterSeq, TimeoutMS: w.Timeout.Milliseconds(), Max: w.Max}, &res) - if err != nil { - return err - } - if res.Superseded { - // Another waiter took over (e.g. this collector restarted elsewhere); keep going. - return nil - } - if res.Instance != w.instance || res.Overflow { - reason := "instance" - if res.Overflow && res.Instance == w.instance { - reason = "overflow" - } - return w.resync(ctx, res, reason) - } - for _, ev := range res.Events { - seq := toInt64(ev["seq"]) - if seq <= w.afterSeq { - continue - } - if err := w.put(res.Instance, seq, ev); err != nil { - return err - } - w.afterSeq = seq - } - if res.Seq > w.afterSeq && len(res.Events) == 0 { - // An empty answer reports the node's current seq: nothing newer is pending. - w.afterSeq = res.Seq - } - return nil -} - -// resync handles a node restart (instance change) or a dropped event (overflow): the full -// jc_ha_status goes to the arbiter as one ha_resync event, and the loop continues from its seq. -func (w *Worker) resync(ctx context.Context, res waitResult, reason string) error { - callCtx, cancel := context.WithTimeout(ctx, httpSlack) - defer cancel() - var status map[string]any - if err := w.RPC.Call(callCtx, "jc_ha_status", nil, &status); err != nil { - return err - } - instance := res.Instance - if s, ok := status["instance"].(string); ok && s != "" { - instance = s - } - seq := toInt64(status["seq"]) - if seq == 0 { - seq = res.Seq - } - payload := map[string]any{ - "status": statusResync, "instance": instance, "seq": seq, - "reason": reason, "ha_status": status, - } - if err := w.put(instance, seq, payload); err != nil { - return err - } - w.Log.Info("resynced", "node", w.NodeID, "reason", reason, "instance", instance, "seq", seq) - w.instance, w.afterSeq = instance, seq - return nil -} - -func (w *Worker) put(instance string, seq int64, payload map[string]any) error { - if _, ok := payload["instance"]; !ok { - payload["instance"] = instance - } - ev := Event{ClusterID: w.ClusterID, NodeID: w.NodeID, Instance: instance, Seq: seq, - ReceivedAt: w.Now().UnixMilli(), Payload: payload} - if err := w.Store.PutEvent(ev); err != nil { - return errors.Join(errors.New("queue event"), err) - } - return nil -} - -func toInt64(v any) int64 { - switch x := v.(type) { - case float64: - return int64(x) - case int64: - return x - case int: - return int64(x) - case string: - n, _ := strconv.ParseInt(x, 10, 64) - return n - } - return 0 -} diff --git a/tools/event-collector/worker_test.go b/tools/event-collector/worker_test.go deleted file mode 100644 index ac4760d071..0000000000 --- a/tools/event-collector/worker_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package main - -import ( - "context" - "errors" - "testing" - "time" -) - -func newWorker(rpc Caller, st Store) *Worker { - return &Worker{ClusterID: "cl", NodeID: "n1", RPC: rpc, Store: st, Now: fixedNow, - Timeout: 50 * time.Millisecond} -} - -func ev(seq int, status string) map[string]any { - return map[string]any{"seq": seq, "event_type": "device_status", "jm_vuid": 3, "status": status} -} - -func key(instance string, seq int64) string { - return Event{ClusterID: "cl", NodeID: "n1", Instance: instance, Seq: seq}.Key() -} - -func TestFirstCallResyncsThenQueuesEventsInOrder(t *testing.T) { - rpc, st := &scripted{}, newMemStore() - rpc.push("jc_wait_events", waitResult{Instance: "i1", Seq: 40}) - rpc.push("jc_ha_status", map[string]any{"instance": "i1", "seq": 40, "epoch": 7}) - rpc.push("jc_wait_events", waitResult{Instance: "i1", Seq: 42, - Events: []map[string]any{ev(41, "remote_jm_unhealthy"), ev(42, "ha_hold_started")}}) - w := newWorker(rpc, st) - for i := 0; i < 2; i++ { - if err := w.Once(context.Background()); err != nil { - t.Fatal(err) - } - } - if st.count() != 3 { - t.Fatalf("want resync + 2 events, got %d", st.count()) - } - resync := st.events[key("i1", 40)] - if resync.Payload["status"] != statusResync || resync.Payload["reason"] != "instance" { - t.Fatalf("resync payload %v", resync.Payload) - } - args := rpc.waitArgs() - if args[0].Instance != "" || args[0].AfterSeq != 0 || args[1].Instance != "i1" || args[1].AfterSeq != 40 { - t.Fatalf("acknowledgements %+v", args) - } - if w.afterSeq != 42 { - t.Fatalf("afterSeq %d", w.afterSeq) - } - got := st.events[key("i1", 41)] - if got.Payload["instance"] != "i1" || got.ReceivedAt != fixedNow().UnixMilli() { - t.Fatalf("event %+v", got) - } -} - -func TestRedeliveredEventsAreSkippedAndEmptyAnswerAdvancesSeq(t *testing.T) { - rpc, st := &scripted{}, newMemStore() - w := newWorker(rpc, st) - w.instance, w.afterSeq = "i1", 42 - rpc.push("jc_wait_events", waitResult{Instance: "i1", Seq: 43, Events: []map[string]any{ev(42, "x"), ev(43, "y")}}) - rpc.push("jc_wait_events", waitResult{Instance: "i1", Seq: 50}) - _ = w.Once(context.Background()) - _ = w.Once(context.Background()) - if st.count() != 1 || w.afterSeq != 50 { - t.Fatalf("count %d afterSeq %d", st.count(), w.afterSeq) - } -} - -func TestOverflowResyncs(t *testing.T) { - rpc, st := &scripted{}, newMemStore() - w := newWorker(rpc, st) - w.instance, w.afterSeq = "i1", 10 - rpc.push("jc_wait_events", waitResult{Instance: "i1", Seq: 2000, Overflow: true}) - rpc.push("jc_ha_status", map[string]any{"instance": "i1", "seq": 2000}) - if err := w.Once(context.Background()); err != nil { - t.Fatal(err) - } - e := st.events[key("i1", 2000)] - if e.Payload["reason"] != "overflow" || w.afterSeq != 2000 { - t.Fatalf("payload %v afterSeq %d", e.Payload, w.afterSeq) - } -} - -func TestNodeRestartResyncsWithTheNewInstance(t *testing.T) { - rpc, st := &scripted{}, newMemStore() - w := newWorker(rpc, st) - w.instance, w.afterSeq = "old", 99 - rpc.push("jc_wait_events", waitResult{Instance: "new", Seq: 3}) - rpc.push("jc_ha_status", map[string]any{"instance": "new", "seq": 3}) - _ = w.Once(context.Background()) - if w.instance != "new" || w.afterSeq != 3 { - t.Fatalf("instance %q afterSeq %d", w.instance, w.afterSeq) - } -} - -func TestSupersededAnswerChangesNothing(t *testing.T) { - rpc, st := &scripted{}, newMemStore() - w := newWorker(rpc, st) - w.instance, w.afterSeq = "i1", 5 - rpc.push("jc_wait_events", waitResult{Instance: "i1", Seq: 9, Superseded: true}) - _ = w.Once(context.Background()) - if st.count() != 0 || w.afterSeq != 5 { - t.Fatalf("count %d afterSeq %d", st.count(), w.afterSeq) - } -} - -func TestFailedStoreWriteKeepsTheAcknowledgement(t *testing.T) { - rpc, st := &scripted{}, newMemStore() - w := newWorker(rpc, st) - w.instance, w.afterSeq = "i1", 1 - st.putErr = errors.New("fdb down") - rpc.push("jc_wait_events", waitResult{Instance: "i1", Seq: 2, Events: []map[string]any{ev(2, "x")}}) - if err := w.Once(context.Background()); err == nil { - t.Fatal("want an error") - } - if w.afterSeq != 1 { - t.Fatalf("afterSeq advanced to %d: the event would be lost", w.afterSeq) - } -} - -func TestRunBacksOffDoublingToTheCap(t *testing.T) { - rpc, st := &scripted{}, newMemStore() - for i := 0; i < 9; i++ { - rpc.push("jc_wait_events", errDown) - } - var sleeps []time.Duration - ctx, cancel := context.WithCancel(context.Background()) - w := newWorker(rpc, st) - w.Sleep = func(_ context.Context, d time.Duration) error { - sleeps = append(sleeps, d) - if len(sleeps) == 9 { - cancel() - return context.Canceled - } - return nil - } - w.Run(ctx) - want := []time.Duration{100, 200, 400, 800, 1600, 3200, 5000, 5000, 5000} - for i, d := range want { - if sleeps[i] != d*time.Millisecond { - t.Fatalf("sleep %d = %v, want %v (%v)", i, sleeps[i], d*time.Millisecond, sleeps) - } - } -} - -func TestEventKeyMatchesThePythonModel(t *testing.T) { - e := Event{ClusterID: "c", NodeID: "n", Instance: "i", Seq: 5} - if e.Key() != "object/ArbitrationEvent/c/n/i/00000000000000000005" { - t.Fatal(e.Key()) - } - r := e.Record() - if r["name"] != "ArbitrationEvent" || r["object_type"] != "object" { - t.Fatalf("%v", r) - } -} From 09d54e452cf95601e1db0bf90ec06cb8579cf8ca Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 20 Nov 2026 17:21:04 +0100 Subject: [PATCH 2/2] fix(two-node): retry a failed event-collector store write instead of dying _collect only caught RPCException, so a failed arbitration_events.queue_event (FDB write) escaped uncaught and killed the worker thread -- it only came back on the next _reconcile tick, via a full resync, instead of retrying the same (instance, after_seq) in place the way the Go collector it replaces did on any error. Catch it the same way. Also ports the Go worker/supervisor coverage nothing carried over for the rewrite: a failed store write doesn't advance after_seq and is safe to redeliver, the collector thread survives one and keeps going, an unchanged topology leaves a running worker alone, a node's address change takes effect without restarting its worker, the ArbitrationEvent key layout (docs/models/arbitration.py) now that only this model writes it, and the idle/active rescan cadence end-to-end through main(). --- .../services/two_node_event_collector.py | 11 + .../test_two_node_event_collector.py | 239 ++++++++++++++++++ .../test_two_node_event_collector_backoff.py | 85 +++++++ 3 files changed, 335 insertions(+) create mode 100644 tests/unit/test_two_node_event_collector_backoff.py diff --git a/simplyblock_core/services/two_node_event_collector.py b/simplyblock_core/services/two_node_event_collector.py index 323c53f5cc..a630a09f9e 100644 --- a/simplyblock_core/services/two_node_event_collector.py +++ b/simplyblock_core/services/two_node_event_collector.py @@ -103,6 +103,17 @@ def _collect(node_id: str, cluster_id: str, stop: threading.Event) -> None: logger.warning(f"jc_wait_events failed on {node_id}: {e}; retry in {backoff:.1f}s") stop.wait(backoff) backoff = min(backoff * 2, BACKOFF_MAX_S) + except Exception as e: + # A store write (arbitration_events.queue_event) can fail for reasons + # that have nothing to do with the RPC transport (FDB unavailable, + # a conflict). after_seq is only advanced once the write that + # covers it succeeds (see _poll_once), so retrying here with the + # same (instance, after_seq) keeps at-least-once delivery instead + # of letting the thread die -- the collector would otherwise sit + # dead until the next _reconcile tick restarts it from scratch. + logger.warning(f"Collector error on {node_id}: {e}; retry in {backoff:.1f}s") + stop.wait(backoff) + backoff = min(backoff * 2, BACKOFF_MAX_S) logger.info(f"Stopping two-node event collector on node: {node_id}") diff --git a/tests/integration/test_two_node_event_collector.py b/tests/integration/test_two_node_event_collector.py index 56bd3199d2..3554bb5a91 100644 --- a/tests/integration/test_two_node_event_collector.py +++ b/tests/integration/test_two_node_event_collector.py @@ -4,6 +4,7 @@ """ import threading +import time import uuid as _uuid import pytest @@ -213,3 +214,241 @@ def test_reconcile_skips_a_removed_node(db, ids, monkeypatch): assert nid not in collector._workers finally: collector._workers.clear() + + +def test_poll_once_store_failure_does_not_advance_seq_and_is_safe_to_retry(db, ids, monkeypatch): + """A failed ArbitrationEvent write (FDB unavailable, a conflict) must not + advance after_seq -- the event has to be re-deliverable from the same + (instance, after_seq) the caller already held, same as Go's + TestFailedStoreWriteKeepsTheAcknowledgement.""" + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + rpc = FakeRPC() + event = {"seq": 5, "status": "ha_solo", "jm_vuid": 3} + rpc.wait_responses = [{"instance": "i1", "seq": 5, "overflow": False, "events": [event]}] + + original_queue_event = collector.arbitration_events.queue_event + monkeypatch.setattr( + collector.arbitration_events, "queue_event", + lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("fdb write failed"))) + with pytest.raises(RuntimeError): + collector._poll_once(rpc, cid, nid, "i1", 0) + assert db.get_arbitration_events(cid) == [] + + # Same (instance, after_seq) as before the failed attempt: the server + # would redeliver the same event, since nothing acknowledged it. + monkeypatch.setattr(collector.arbitration_events, "queue_event", original_queue_event) + rpc.wait_responses = [{"instance": "i1", "seq": 5, "overflow": False, "events": [dict(event)]}] + instance, after_seq = collector._poll_once(rpc, cid, nid, "i1", 0) + assert (instance, after_seq) == ("i1", 5) + events = db.get_arbitration_events(cid) + assert len(events) == 1 + assert events[0].seq == 5 + + +def test_collect_thread_survives_a_transient_store_write_failure(db, ids, monkeypatch): + """The collector thread must not die on a store-write error: it should + back off and retry in place, the same contract a transport error gets, + rather than rely on the supervisor to notice and restart it from scratch + on the next reconcile tick.""" + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + + class _ResyncRPC: + """Always reports an instance the worker hasn't seen yet, so every + poll (while the worker's local ``instance`` stays "") tries a + resync -- exactly the retry this test needs to observe.""" + + def jc_wait_events(self, instance, after_seq, timeout_ms=20000, max_events=256): + return {"instance": "i1", "seq": 1, "overflow": False, "events": []} + + def jc_ha_status(self): + return {"instance": "i1", "seq": 1, "lvs": []} + + rpc = _ResyncRPC() + monkeypatch.setattr(StorageNode, "rpc_client", lambda self, **kw: rpc) + + calls = {"n": 0} + original_queue_event = collector.arbitration_events.queue_event + + def flaky_queue_event(*a, **kw): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("fdb write failed") + return original_queue_event(*a, **kw) + + monkeypatch.setattr(collector.arbitration_events, "queue_event", flaky_queue_event) + monkeypatch.setattr(collector, "BACKOFF_MIN_S", 0.01) + monkeypatch.setattr(collector, "BACKOFF_MAX_S", 0.02) + + stop = threading.Event() + thread = threading.Thread(target=collector._collect, args=(nid, cid, stop), daemon=True) + thread.start() + try: + for _ in range(200): + if calls["n"] >= 2: + break + time.sleep(0.02) + assert calls["n"] >= 2, "the collector thread died instead of retrying" + assert thread.is_alive() + finally: + stop.set() + thread.join(timeout=2) + events = db.get_arbitration_events(cid) + assert len(events) == 1 + assert events[0].payload["reason"] == "instance" + + +def test_reconcile_does_not_restart_an_unchanged_worker(db, ids, monkeypatch): + """A reconcile tick with no topology change must leave a running worker's + thread in place -- restarting it would drop its in-memory (instance, + after_seq) and force a needless resync.""" + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + + def fake_collect(node_id, cluster_id, stop): + stop.wait() + + monkeypatch.setattr(collector, "_collect", fake_collect) + collector._workers.clear() + try: + collector._reconcile() + assert nid in collector._workers + thread_before = collector._workers[nid][0] + + collector._reconcile() + assert collector._workers[nid][0] is thread_before + assert thread_before.is_alive() + finally: + for thread, stop in collector._workers.values(): + stop.set() + thread.join(timeout=2) + collector._workers.clear() + + +def test_collect_picks_up_a_node_address_change_without_a_worker_restart(db, ids, monkeypatch): + """The worker re-fetches the StorageNode every poll and builds a fresh RPC + client from it (see _collect), so a changed mgmt_ip takes effect on the + next poll -- the supervisor does not need to recreate the thread for it.""" + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + + addresses: list[tuple] = [] + + class _SlowRPC: + def jc_wait_events(self, instance, after_seq, timeout_ms=20000, max_events=256): + time.sleep(0.02) + return {"instance": instance, "seq": after_seq, "overflow": False, "events": []} + + def jc_ha_status(self): + return {} + + def fake_rpc_client(self, **kwargs): + addresses.append((self.mgmt_ip, self.rpc_port)) + return _SlowRPC() + + monkeypatch.setattr(StorageNode, "rpc_client", fake_rpc_client) + collector._workers.clear() + try: + collector._reconcile() + assert nid in collector._workers + thread_before = collector._workers[nid][0] + + for _ in range(200): + if addresses: + break + time.sleep(0.02) + assert addresses, "the worker never polled" + + node = db.get_storage_node_by_id(nid) + node.mgmt_ip = "10.0.0.9" + node.write_to_db(db.kv_store) + + # A reconcile tick with the node's cluster membership unchanged must + # not touch the running worker. + collector._reconcile() + assert collector._workers[nid][0] is thread_before + + for _ in range(200): + if any(addr == ("10.0.0.9", 8080) for addr in addresses): + break + time.sleep(0.02) + else: + pytest.fail(f"worker never used the new address: {addresses}") + assert collector._workers[nid][0] is thread_before + finally: + for thread, stop in collector._workers.values(): + stop.set() + thread.join(timeout=2) + collector._workers.clear() + + +def test_arbitration_event_key_matches_the_pinned_fdb_layout(db, ids): + """Pins the on-disk key format (docs: models/arbitration.py), the same + guarantee Go's TestEventKeyMatchesThePythonModel gave when both + implementations wrote the same store -- now that only this model writes + it, nothing else protects the layout the arbiter's prefix-reads depend on.""" + cid, nid = ids + _cluster(cid).write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + rpc = FakeRPC() + rpc.wait_responses = [{"instance": "i1", "seq": 5, "overflow": False, + "events": [{"seq": 5, "status": "ha_solo", "jm_vuid": 3}]}] + collector._poll_once(rpc, cid, nid, "i1", 0) + + expected_key = f"object/ArbitrationEvent/{cid}/{nid}/i1/{5:020d}".encode() + raw = db.kv_store[expected_key] + assert raw is not None, "ArbitrationEvent key layout changed" + import json + stored = json.loads(bytes(raw)) + assert stored["cluster_id"] == cid + assert stored["node_id"] == nid + assert stored["instance"] == "i1" + assert stored["seq"] == 5 + + events = db.get_arbitration_events(cid) + assert len(events) == 1 and events[0].seq == 5 + + +def test_idle_and_active_rescan_cadence_across_a_topology_change(db, ids, monkeypatch): + """TWO_NODE_IDLE_RESCAN_S while nothing is arbitrated, TWO_NODE_COLLECTOR_RECONCILE_S + while a worker runs, and back to idle once arbitration is turned off -- + driven through the real main() loop and a real reconcile against FDB.""" + from simplyblock_core import constants + + cid, nid = ids + cluster = _cluster(cid, arbitration=False) + cluster.write_to_db(db.kv_store) + + monkeypatch.setattr(collector, "_collect", lambda *a: None) + sleeps: list[float] = [] + + def fake_sleep(duration): + sleeps.append(duration) + if len(sleeps) == 1: + cluster.two_node_arbitration = True + cluster.write_to_db(db.kv_store) + _node(nid, cid).write_to_db(db.kv_store) + elif len(sleeps) == 2: + cluster.two_node_arbitration = False + cluster.write_to_db(db.kv_store) + elif len(sleeps) >= 3: + raise SystemExit + + monkeypatch.setattr(collector.time, "sleep", fake_sleep) + collector._workers.clear() + try: + with pytest.raises(SystemExit): + collector.main() + finally: + collector._workers.clear() + + assert sleeps == [ + constants.TWO_NODE_IDLE_RESCAN_S, + constants.TWO_NODE_COLLECTOR_RECONCILE_S, + constants.TWO_NODE_IDLE_RESCAN_S, + ] diff --git a/tests/unit/test_two_node_event_collector_backoff.py b/tests/unit/test_two_node_event_collector_backoff.py new file mode 100644 index 0000000000..b4a9816df4 --- /dev/null +++ b/tests/unit/test_two_node_event_collector_backoff.py @@ -0,0 +1,85 @@ +"""Two-node event collector: backoff on a repeated RPC or store error, and its +reset on the next success (docs/design/two-node-arbitration.md, section 4; +contract 100 ms doubling to 5 s, same as Go's TestRunBacksOffDoublingToTheCap). +Pure logic, no DB: the module-level ``db`` is faked. +""" + +import threading + +import pytest + +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCException +from simplyblock_core.services import two_node_event_collector as collector + + +class _FakeDB: + def __init__(self, node): + self._node = node + + def get_storage_node_by_id(self, node_id): + return self._node + + +class _AlwaysFailingNode: + status = StorageNode.STATUS_ONLINE + + def rpc_client(self, **kwargs): + raise RPCException("down") + + +class _FlakyNode: + """Fails every call except the third, which succeeds with no new events -- + enough to let ``_collect`` reset its backoff before failing again.""" + + status = StorageNode.STATUS_ONLINE + + def __init__(self): + self.calls = 0 + + def rpc_client(self, **kwargs): + self.calls += 1 + if self.calls == 3: + return _SteadyRPC() + raise RPCException("down") + + +class _SteadyRPC: + def jc_wait_events(self, instance, after_seq, timeout_ms=20000, max_events=256): + return {"instance": instance, "seq": after_seq, "overflow": False, "events": []} + + +def test_backoff_doubles_to_the_cap_then_holds(monkeypatch): + monkeypatch.setattr(collector, "db", _FakeDB(_AlwaysFailingNode())) + stop = threading.Event() + sleeps: list[float] = [] + + def fake_wait(timeout): + sleeps.append(timeout) + if len(sleeps) == 9: + stop.set() + + monkeypatch.setattr(stop, "wait", fake_wait) + collector._collect("n1", "cl", stop) + + want = [0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 5.0, 5.0, 5.0] + assert sleeps == pytest.approx(want) + + +def test_backoff_resets_after_a_success(monkeypatch): + node = _FlakyNode() + monkeypatch.setattr(collector, "db", _FakeDB(node)) + stop = threading.Event() + sleeps: list[float] = [] + + def fake_wait(timeout): + sleeps.append(timeout) + if len(sleeps) == 4: + stop.set() + + monkeypatch.setattr(stop, "wait", fake_wait) + collector._collect("n1", "cl", stop) + + # calls 1, 2 fail (0.1, 0.2); call 3 succeeds and resets backoff; + # calls 4, 5 fail again starting back at 0.1. + assert sleeps == pytest.approx([0.1, 0.2, 0.1, 0.2])