From 5223312de1d3f819c70923ca44b88050453e8991 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Mon, 22 Jun 2026 03:07:52 -0700 Subject: [PATCH 01/25] feat(dash-spv-bench): minimal single-scenario sync benchmark --- Cargo.toml | 2 +- dash-spv-bench/.gitignore | 21 ++ dash-spv-bench/Cargo.toml | 24 ++ dash-spv-bench/Dockerfile | 23 ++ dash-spv-bench/bench-scenarios.sh | 88 +++++ dash-spv-bench/run.sh | 335 ++++++++++++++++++ dash-spv-bench/scenario.example.yml | 55 +++ dash-spv-bench/scenarios/congested-2core.yml | 5 + dash-spv-bench/scenarios/ideal-1-peer.yml | 5 + dash-spv-bench/scenarios/ideal-3-peers.yml | 5 + dash-spv-bench/scenarios/ideal-5-peers.yml | 5 + .../scenarios/no-lag-ideal-1-peer.yml | 5 + .../scenarios/no-lag-ideal-3-peers.yml | 5 + .../scenarios/no-lag-ideal-5-peers.yml | 5 + dash-spv-bench/scenarios/one-bad-peer.yml | 6 + dash-spv-bench/scenarios/real-mainnet.yml | 4 + dash-spv-bench/scenarios/real-testnet.yml | 4 + dash-spv-bench/snapshot-chain.sh | 34 ++ dash-spv-bench/src/main.rs | 168 +++++++++ dash-spv-bench/src/metrics.rs | 161 +++++++++ dash-spv/src/lib.rs | 1 + dash-spv/src/timer.rs | 148 ++++++++ 22 files changed, 1108 insertions(+), 1 deletion(-) create mode 100644 dash-spv-bench/.gitignore create mode 100644 dash-spv-bench/Cargo.toml create mode 100644 dash-spv-bench/Dockerfile create mode 100755 dash-spv-bench/bench-scenarios.sh create mode 100755 dash-spv-bench/run.sh create mode 100644 dash-spv-bench/scenario.example.yml create mode 100644 dash-spv-bench/scenarios/congested-2core.yml create mode 100644 dash-spv-bench/scenarios/ideal-1-peer.yml create mode 100644 dash-spv-bench/scenarios/ideal-3-peers.yml create mode 100644 dash-spv-bench/scenarios/ideal-5-peers.yml create mode 100644 dash-spv-bench/scenarios/no-lag-ideal-1-peer.yml create mode 100644 dash-spv-bench/scenarios/no-lag-ideal-3-peers.yml create mode 100644 dash-spv-bench/scenarios/no-lag-ideal-5-peers.yml create mode 100644 dash-spv-bench/scenarios/one-bad-peer.yml create mode 100644 dash-spv-bench/scenarios/real-mainnet.yml create mode 100644 dash-spv-bench/scenarios/real-testnet.yml create mode 100755 dash-spv-bench/snapshot-chain.sh create mode 100644 dash-spv-bench/src/main.rs create mode 100644 dash-spv-bench/src/metrics.rs create mode 100644 dash-spv/src/timer.rs diff --git a/Cargo.toml b/Cargo.toml index f885763f1..63049fbea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["dash", "dash-network", "hashes", "internals", "fuzz", "rpc-client", "rpc-json", "rpc-integration-test", "key-wallet", "key-wallet-manager", "key-wallet-ffi", "dash-spv", "dash-spv-ffi", "git-state", "dash-network-seeds", "masternode-seeds-fetcher"] +members = ["dash", "dash-network", "hashes", "internals", "fuzz", "rpc-client", "rpc-json", "rpc-integration-test", "key-wallet", "key-wallet-manager", "key-wallet-ffi", "dash-spv", "dash-spv-ffi", "git-state", "dash-network-seeds", "masternode-seeds-fetcher", "dash-spv-bench"] resolver = "2" [workspace.package] diff --git a/dash-spv-bench/.gitignore b/dash-spv-bench/.gitignore new file mode 100644 index 000000000..e131dfb23 --- /dev/null +++ b/dash-spv-bench/.gitignore @@ -0,0 +1,21 @@ +bench-results/ +bench-storage/ +profiles/ + +*.lock + +# Local bench config (copied from .env.example). +.env + +# Persistent testnet chain snapshot grown by snapshot-chain.sh (multi-GB; never committed). +chain-data/ + +# Transient per-peer CoW clones created by run.sh + the file that remembers their dir. +.bench-clones.* +.clonedir + +# Scenario matrix outputs (bench-scenarios.sh): generated reports/logs/compose, and the +# bootstrapped yq binary. Reports are meant to be copied out, not committed. +.scenario.*.yml +results/ +.bin/ diff --git a/dash-spv-bench/Cargo.toml b/dash-spv-bench/Cargo.toml new file mode 100644 index 000000000..c0201922e --- /dev/null +++ b/dash-spv-bench/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "dash-spv-bench" +version = { workspace = true } +edition = "2021" +license = "MIT" +publish = false + +[dependencies] +dash-spv = { path = "../dash-spv" } +dashcore = { path = "../dash", features = ["serde", "core-block-hash-use-x11", "message_verification", "bls", "quorum_validation"] } +key-wallet = { path = "../key-wallet" } +key-wallet-manager = { path = "../key-wallet-manager", features = ["parallel-filters"] } + +tokio = { version = "1.0", features = ["full"] } + +anyhow = "1.0" +tempfile = "3.0" + +tracing = "0.1" +tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } + +[[bin]] +name = "dash-spv-bench" +path = "src/main.rs" diff --git a/dash-spv-bench/Dockerfile b/dash-spv-bench/Dockerfile new file mode 100644 index 000000000..3cdaca673 --- /dev/null +++ b/dash-spv-bench/Dockerfile @@ -0,0 +1,23 @@ +FROM debian:bookworm-slim + +ARG DASHVERSION=23.1.4 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl iproute2 \ + && rm -rf /var/lib/apt/lists/* + +# Download the Linux dashd matching the host CPU arch (amd64 -> x86_64, arm64 -> aarch64), +# keeping the same release used by contrib/setup-dashd.py. +RUN set -eux; \ + arch="$(dpkg --print-architecture)"; \ + case "$arch" in \ + amd64) dlarch=x86_64 ;; \ + arm64) dlarch=aarch64 ;; \ + *) echo "unsupported arch $arch" >&2; exit 1 ;; \ + esac; \ + url="https://github.com/dashpay/dash/releases/download/v${DASHVERSION}/dashcore-${DASHVERSION}-${dlarch}-linux-gnu.tar.gz"; \ + curl -fsSL "$url" -o /tmp/dashcore.tgz; \ + tar -xzf /tmp/dashcore.tgz -C /tmp; \ + cp "/tmp/dashcore-${DASHVERSION}/bin/dashd" /usr/local/bin/dashd; \ + rm -rf /tmp/dashcore*; \ + dashd --version | head -1 diff --git a/dash-spv-bench/bench-scenarios.sh b/dash-spv-bench/bench-scenarios.sh new file mode 100755 index 000000000..7e1ffd744 --- /dev/null +++ b/dash-spv-bench/bench-scenarios.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# dash-spv-bench scenario matrix (LOCAL mode only). +# +# ./bench-scenarios.sh [-d scenarios] +# +# Thin loop over scenario files: for each scenarios/*.yml it calls `run.sh scenario ` +# once per its `repeats:` count, captures the timings run.sh prints, and writes an organized +# Markdown report (median over repeats). All the real work — compose generation, bring-up, +# CPU pinning, teardown — lives in run.sh; this only orchestrates and aggregates. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +cd "${SCRIPT_DIR}" + +SCN_DIR="${SCRIPT_DIR}/scenarios" +RUN_CMD="${RUN_CMD:-${SCRIPT_DIR}/run.sh}" # overridable for testing the loop without docker +while [ $# -gt 0 ]; do + case "$1" in + -d | --dir) SCN_DIR="$2"; shift 2 ;; + -h | --help) sed -n '3,10p' "$0"; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 1 ;; + esac +done + +shopt -s nullglob +FILES=("${SCN_DIR}"/*.yml) +[ "${#FILES[@]}" -gt 0 ] || { echo "Error: no scenarios (*.yml) in ${SCN_DIR}" >&2; exit 1; } + +RUN_TS="$(date +%Y%m%d-%H%M%S)" +OUT_DIR="${SCRIPT_DIR}/results/${RUN_TS}" +mkdir -p "${OUT_DIR}" +REPORT="${OUT_DIR}/report.md" +TSV="${OUT_DIR}/results.tsv" +: >"${TSV}" + +metric() { awk -F':[[:space:]]*' -v k="$2" '$1==k {gsub(/[[:space:]]+$/,"",$2); print $2; exit}' "$1"; } +median() { sort -n | awk '{a[NR]=$1} END{if(NR==0){print"-";exit} print (NR%2)?a[(NR+1)/2]:int((a[NR/2]+a[NR/2+1])/2)}'; } + +for f in "${FILES[@]}"; do + name="$(basename "${f}" .yml)" + repeats="$(awk '/^repeats:/{print $2; exit}' "${f}")"; repeats="${repeats:-1}" + echo "===== scenario '${name}' × ${repeats} =====" + for r in $(seq 1 "${repeats}"); do + echo "-- ${name} repeat ${r}/${repeats}" + log="${OUT_DIR}/${name}.r${r}.log" + "${RUN_CMD}" "${f}" >"${log}" 2>&1 || echo " (run.sh exited non-zero; see ${log})" + + # Scenario metadata comes straight from run.sh's own echo lines, so we never re-parse the YAML. + meta="$(grep -m1 "==> scenario '" "${log}" || true)" + cpus="$(sed -n "s/.*cpus=\([^,]*\),.*/\1/p" <<<"${meta}")" + blocks="$(sed -n "s/.*blocks=\([0-9]*\).*/\1/p" <<<"${meta}")" + peers="$(sed -n "s/.*\[\(.*\)\], cpus=.*/\1/p" <<<"${meta}")" + desc="$(grep -m1 "==> description:" "${log}" | sed 's/.*==> description: //' || true)" + + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${name}" "${cpus}" "${blocks}" "${peers}" "${r}" \ + "$(metric "${log}" completed)" "$(metric "${log}" total_ms)" \ + "$(metric "${log}" block_headers_ms)" "$(metric "${log}" filter_headers_ms)" \ + "$(metric "${log}" filters_ms)" "$(metric "${log}" filter_hdr_lag_ms)" "${desc}" \ + >>"${TSV}" + done +done + +# --- report --------------------------------------------------------------------------------- +{ + echo "# dash-spv-bench scenario report" + echo + echo "- **Date:** $(date -u '+%Y-%m-%d %H:%M:%SZ')" + echo "- **Host:** $(nproc) cores — $(grep -m1 'model name' /proc/cpuinfo 2>/dev/null | cut -d: -f2 | sed 's/^ //' || uname -m)" + echo "- **git:** $(git -C "${REPO_ROOT}" rev-parse --abbrev-ref HEAD 2>/dev/null)@$(git -C "${REPO_ROOT}" rev-parse --short HEAD 2>/dev/null)" + echo "- **Scenarios:** \`$(basename "${SCN_DIR}")/\` (${#FILES[@]} files)" + echo + echo "Timings are the **median** over each scenario's repeats (ms)." + echo + echo "| scenario | description | cpus | blocks | peers | completed | total | block_hdrs | filter_hdrs | filters | fh_lag |" + echo "|---|---|---|---|---|---|---|---|---|---|---|" + for name in $(cut -f1 "${TSV}" | awk '!seen[$0]++'); do + row() { awk -F'\t' -v n="${name}" '$1==n{print $'"$1"'; exit}' "${TSV}"; } + med() { awk -F'\t' -v n="${name}" '$1==n && $'"$1"'!="" && $'"$1"'!="-"{print $'"$1"'}' "${TSV}" | median; } + echo "| ${name} | $(row 12) | $(row 2) | $(row 3) | $(row 4) | $(row 6) | $(med 7) | $(med 8) | $(med 9) | $(med 10) | $(med 11) |" + done + echo + echo "Raw per-repeat data: \`results.tsv\`; per-run logs alongside this report." +} >"${REPORT}" + +echo "==> wrote ${REPORT}" diff --git a/dash-spv-bench/run.sh b/dash-spv-bench/run.sh new file mode 100755 index 000000000..8c5cbde42 --- /dev/null +++ b/dash-spv-bench/run.sh @@ -0,0 +1,335 @@ +#!/bin/bash +# +# dash-spv benchmark driver — runs ONE scenario. Self-contained and idempotent. +# +# ./run.sh Run the scenario (local or testnet per its `mode:`). +# ./run.sh --flame Same, under the sampler (perf on Linux, sample on macOS) +# -> profiles/flamegraph.svg +# +# Everything is configured by the scenario file — there is NO .env. See scenario.example.yml +# for every field. Local mode needs only docker (it builds the snapshot and the peers in +# containers); testnet mode needs nothing extra. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +COMPOSE_FILE="" # local mode generates one here; deleted on exit +IMAGE="dash-spv-bench/dashd:latest" +PROJECT="spv-bench" +STATE="${SCRIPT_DIR}/.clonedir" +FLAME_SVG="${SCRIPT_DIR}/profiles/flamegraph.svg" +CHAIN_DIR="${SCRIPT_DIR}/chain-data" + +cd "${SCRIPT_DIR}" + +# --- argument parsing ----------------------------------------------------------------------- +FLAME=0 +SCN_FILE="" +while [ $# -gt 0 ]; do + case "$1" in + --flame) FLAME=1; shift ;; + -h | --help) sed -n '3,11p' "$0"; exit 0 ;; + -*) echo "unknown flag: $1" >&2; exit 1 ;; + *) SCN_FILE="$1"; shift ;; + esac +done +[ -n "${SCN_FILE}" ] || { echo "usage: $0 [--flame]" >&2; exit 1; } +[ -f "${SCN_FILE}" ] || { echo "Error: scenario file not found: ${SCN_FILE}" >&2; exit 1; } + +# --- yq (YAML without python) --------------------------------------------------------------- +YQ_VERSION="${YQ_VERSION:-v4.44.6}" +ensure_yq() { + if command -v yq >/dev/null 2>&1; then YQ=yq; return 0; fi + local bin="${SCRIPT_DIR}/.bin/yq" + if [ ! -x "${bin}" ]; then + mkdir -p "${SCRIPT_DIR}/.bin" + local os arch + os="$(uname -s | tr '[:upper:]' '[:lower:]')"; arch="$(uname -m)" + case "${arch}" in x86_64 | amd64) arch=amd64 ;; aarch64 | arm64) arch=arm64 ;; esac + echo "==> fetching yq ${YQ_VERSION} (${os}/${arch}) into .bin/yq" + curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_${os}_${arch}" \ + -o "${bin}" || { echo "Error: could not download yq; install it manually." >&2; exit 1; } + chmod +x "${bin}" + fi + YQ="${bin}" +} +ensure_yq +scn() { "${YQ}" "$1" "${SCN_FILE}"; } # evaluate a yq expression against the scenario file + +# --- scenario -> peer compose (local mode) helpers ------------------------------------------ +# TSV of one peer group's fields: count lat jit loss rate corrupt reorder. +_peer_group() { + "${YQ}" ".peers[$1] | [.count, .latency_ms // 0, .jitter_ms // 0, .loss_pct // 0, .rate_kbit // 0, .corrupt_pct // 0, .reorder_pct // 0] | @tsv" "${SCN_FILE}" +} +build_netem() { + local lat="$1" jit="$2" loss="$3" rate="$4" corrupt="$5" reorder="$6" a="" + [ "${lat}" != 0 ] && { a="delay ${lat}ms"; [ "${jit}" != 0 ] && a="${a} ${jit}ms"; } + [ "${loss}" != 0 ] && a="${a} loss ${loss}%" + [ "${rate}" != 0 ] && a="${a} rate ${rate}kbit" + [ "${corrupt}" != 0 ] && a="${a} corrupt ${corrupt}%" + [ "${reorder}" != 0 ] && a="${a} reorder ${reorder}%" + echo "${a# }" +} +peers_summary() { + local ng g count lat jit loss rate corrupt reorder tag out="" + ng="$(scn '.peers | length')" + for g in $(seq 0 $((ng - 1))); do + read -r count lat jit loss rate corrupt reorder <<<"$(_peer_group "${g}")" + tag="${lat}ms" + [ "${jit}" != 0 ] && tag="${tag}±${jit}" + [ "${loss}" != 0 ] && tag="${tag}/${loss}%loss" + [ "${rate}" != 0 ] && tag="${tag}/${rate}kbit" + out="${out}, ${count}×${tag}" + done + echo "${out#, }" +} +# emit_compose — one dashd service per peer, each with its group's netem. +# Echoes total peer count. `out` MUST live in SCRIPT_DIR so `build: context: .` finds the Dockerfile. +emit_compose() { + local out="$1" peer_cpus="$2" + cat >"${out}" <<'ANCHOR' +# GENERATED for a bench scenario — do not edit; regenerated and deleted each run. +x-dashd-peer: &dashd-peer + image: dash-spv-bench/dashd:latest + build: + context: . + dockerfile: Dockerfile + cap_add: [NET_ADMIN] + entrypoint: ["/bin/sh", "-ec"] + command: + - | + if [ -n "$${NETEM_ARGS:-}" ]; then + tc qdisc add dev eth0 root netem $${NETEM_ARGS} \ + && echo "netem: $${NETEM_ARGS}" || echo "WARNING: netem failed (NET_ADMIN/sch_netem?)" + fi + exec dashd -testnet -datadir=/data -port=19400 -rpcport=19500 -server=1 -daemon=0 \ + -connect=0 -bind=0.0.0.0 -listen=1 -rpcbind=0.0.0.0 -rpcallowip=0.0.0.0/0 \ + -whitelist=0.0.0.0/0 -disablewallet=1 -peerbloomfilters=1 \ + -dbcache=64 -fallbackfee=0.00001 -txindex=0 -addressindex=0 $${FILTER_FLAGS} + +services: +ANCHOR + local ng g count lat jit loss rate corrupt reorder netem n peer=0 + ng="$(scn '.peers | length')" + for g in $(seq 0 $((ng - 1))); do + read -r count lat jit loss rate corrupt reorder <<<"$(_peer_group "${g}")" + netem="$(build_netem "${lat}" "${jit}" "${loss}" "${rate}" "${corrupt}" "${reorder}")" + for n in $(seq 1 "${count}"); do + peer=$((peer + 1)) + cat >>"${out}" <&2; exit 1 ;; esac +export BENCH_MODE="${MODE}" +export CLONE_DIR="${CLONE_DIR:-/nonexistent}" + +BENCH_CPUS="$(scn '.cpus // ""')" +export BENCH_MAX_PEERS="$(scn '.max_peers // ""')" # only if set; else the ClientConfig default +mnem="$(scn '.mnemonic // ""')" +[ -n "${mnem}" ] && export BENCH_MNEMONIC="${mnem}" +DESC="$(scn '.description // ""')" +[ -n "${DESC}" ] && echo "==> description: ${DESC}" + +BLOCKS="" +if [ "${MODE}" = local ]; then + # `blocks` IS the target height: snapshot-chain builds ./chain-data to it, and the client + # syncs the full range genesis..blocks. Default 1M. + BLOCKS="$(scn '.blocks // 1000000')" + export BENCH_HEIGHT="${BLOCKS}" + unset BENCH_START_HEIGHT # local always syncs from genesis +else + # testnet/mainnet: an explicit "host:port,host:port" list, or empty for DNS discovery. + export BENCH_PEERS="$(scn '.peers // [] | join(",")')" + sh="$(scn '.start_height // ""')"; [ -n "${sh}" ] && export BENCH_START_HEIGHT="${sh}" +fi + +# --- CPU pinning: taskset for the binary, complement for the docker peers ------------------- +expand_cpus() { + local part lo hi + local IFS=, + for part in $1; do + case "${part}" in + *-*) lo="${part%-*}"; hi="${part#*-}"; seq "${lo}" "${hi}" ;; + *) echo "${part}" ;; + esac + done +} + +CPU_PREFIX=() +if [ -n "${BENCH_CPUS}" ]; then + command -v taskset >/dev/null 2>&1 || { + echo "Error: cpus is set ('${BENCH_CPUS}') but 'taskset' was not found (Linux/util-linux)." >&2 + exit 1 + } + ncpu="$(nproc)" + bench_cores="$(expand_cpus "${BENCH_CPUS}" | sort -nu)" + for core in ${bench_cores}; do + if [ "${core}" -lt 0 ] || [ "${core}" -ge "${ncpu}" ]; then + echo "Error: cpu ${core} is out of range for this ${ncpu}-core host (valid: 0-$((ncpu - 1)))." >&2 + exit 1 + fi + done + peer_cores="" + for i in $(seq 0 $((ncpu - 1))); do + grep -qxF "${i}" <<<"${bench_cores}" || peer_cores="${peer_cores},${i}" + done + BENCH_PEER_CPUS="${peer_cores#,}" + CPU_PREFIX=(taskset -c "${BENCH_CPUS}") + echo "==> pinning the measured run to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" +fi + +# --- local: generate the peer compose (now that the peer cpuset is known) ------------------- +if [ "${MODE}" = local ]; then + COMPOSE_FILE="$(mktemp "${SCRIPT_DIR}/.scenario.XXXXXX.yml")" + npeers="$(emit_compose "${COMPOSE_FILE}" "${BENCH_PEER_CPUS:-}")" + echo "==> scenario '$(basename "${SCN_FILE}" .yml)': ${npeers} peers [$(peers_summary)], cpus=${BENCH_CPUS:-}, blocks=${BLOCKS}" +fi + +compose() { docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@"; } + +PEER_CONTAINERS="" +peers_ready() { + local c logs recent + for c in ${PEER_CONTAINERS}; do + logs="$(docker logs "${c}" 2>&1)" || return 1 + case "${logs}" in *"init message: Done loading"*) ;; *) return 1 ;; esac + recent="$(docker logs --since 5s "${c}" 2>&1)" + case "${recent}" in *"UpdateTip"*) return 1 ;; esac + done +} +wait_loaded() { + local c logs + for _ in $(seq 1 400); do + local all=1 + for c in "$@"; do + logs="$(docker logs "${c}" 2>&1)" || { all=0; break; } + case "${logs}" in *"init message: Done loading"*) ;; *) all=0; break ;; esac + done + [ "${all}" -eq 1 ] && return 0 + echo -n "."; sleep 3 + done + return 1 +} +# Stop containers and remove CoW clones. Called both at the start of bring_up (to clean prior +# state) and on exit — so it must NOT delete the generated compose (bring_up still needs it); +# that deletion happens only in the exit trap below. +teardown() { + [ -n "${COMPOSE_FILE}" ] && compose down --remove-orphans >/dev/null 2>&1 || true + [ -f "${STATE}" ] && { rm -rf "$(cat "${STATE}")" 2>/dev/null || true; rm -f "${STATE}"; } + rm -rf "${SCRIPT_DIR}"/.bench-clones.* 2>/dev/null || true +} +arm_teardown() { + # On exit: tear down, THEN delete the generated compose (down needs it to still exist). + trap 'teardown; [ -n "${COMPOSE_FILE}" ] && rm -f "${COMPOSE_FILE}"' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + trap 'exit 129' HUP +} +bring_up() { + echo "==> ensuring a clean network" + teardown + docker image inspect "${IMAGE}" >/dev/null 2>&1 || { echo "==> building peer image"; compose build; } + + local services; services="$(compose config --services)" + local n; n="$(echo ${services} | wc -w | tr -d ' ')" + PEER_CONTAINERS="" + for svc in ${services}; do PEER_CONTAINERS="${PEER_CONTAINERS} spv-bench-${svc}"; done + # BENCH_PEERS is derived from the generated peers (host ports 19401..). + local peers_csv="" + for svc in ${services}; do peers_csv="${peers_csv},127.0.0.1:$((19400 + ${svc#dashd}))"; done + export BENCH_PEERS="${peers_csv#,}" + + echo "==> CoW-cloning ${CHAIN_DIR} for ${n} peers (instant)" + local clone_dir; clone_dir="$(mktemp -d "${SCRIPT_DIR}/.bench-clones.XXXXXX")" + echo "${clone_dir}" > "${STATE}" + for svc in ${services}; do + local dst="${clone_dir}/peer${svc#dashd}" + cp -c -R "${CHAIN_DIR}" "${dst}" 2>/dev/null \ + || cp --reflink=auto -R "${CHAIN_DIR}" "${dst}" 2>/dev/null \ + || cp -R "${CHAIN_DIR}" "${dst}" + done + + local batch="${PEER_BATCH:-4}" + echo "==> starting ${n} peers in batches of ${batch}" + local started="" count=0 + for svc in ${services}; do + CLONE_DIR="${clone_dir}" compose up -d "${svc}" >/dev/null 2>&1 + started="${started} spv-bench-${svc}"; count=$((count + 1)) + if [ $((count % batch)) -eq 0 ]; then + echo -n " loaded ${count}/${n} " + wait_loaded ${started} || { echo " timeout loading batch" >&2; exit 1; } + echo " ok" + fi + done + echo -n "==> waiting for all ${n} peers to be ready " + for _ in $(seq 1 600); do + if peers_ready; then echo " ready"; return 0; fi + echo -n "."; sleep 3 + done + echo " timeout" >&2; exit 1 +} + +build_bin() { + echo "==> building bench binary (release + line-table symbols)" + ( cd "${REPO_ROOT}" && CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ + cargo build --release -p dash-spv-bench ) + BIN="${REPO_ROOT}/target/release/dash-spv-bench" +} + +FLAME_TOOL="" +if [ "${FLAME}" -eq 1 ]; then # fail fast on missing profiler deps, before building + if command -v perf >/dev/null; then FLAME_TOOL=perf # Linux + elif command -v /usr/bin/sample >/dev/null; then FLAME_TOOL=sample # macOS + else echo "flame mode needs 'perf' (Linux) or '/usr/bin/sample' (macOS)" >&2; exit 1; fi + [ -f "${HOME}/FlameGraph/flamegraph.pl" ] || \ + git clone --depth 1 https://github.com/brendangregg/FlameGraph "${HOME}/FlameGraph" +fi + +build_bin + +if [ "${MODE}" = local ]; then + arm_teardown + bash "${SCRIPT_DIR}/snapshot-chain.sh" # builds ./chain-data to BENCH_HEIGHT via docker + bring_up +else + echo "==> testnet mode, peers: ${BENCH_PEERS:-}" +fi + +if [ "${FLAME}" -eq 0 ]; then + echo "==> running sync" + "${CPU_PREFIX[@]}" "${BIN}" +elif [ "${FLAME_TOOL}" = perf ]; then + echo "==> running sync under perf" + mkdir -p "$(dirname "${FLAME_SVG}")" + perf record -F 499 -g -o /tmp/bench-perf.data -- "${CPU_PREFIX[@]}" "${BIN}" + perf script -i /tmp/bench-perf.data \ + | "${HOME}/FlameGraph/stackcollapse-perf.pl" \ + | "${HOME}/FlameGraph/flamegraph.pl" --title "dash-spv sync" --colors hot > "${FLAME_SVG}" + echo "==> wrote ${FLAME_SVG}" +else + echo "==> running sync under the sampler" + "${CPU_PREFIX[@]}" "${BIN}" & bpid=$! + sleep 3 + /usr/bin/sample "${bpid}" 2000 1 -file /tmp/bench.sample.txt -mayDie >/dev/null 2>&1 || true + wait "${bpid}" 2>/dev/null || true + mkdir -p "$(dirname "${FLAME_SVG}")" + "${HOME}/FlameGraph/stackcollapse-sample.awk" /tmp/bench.sample.txt \ + | sed -E 's/^Thread_[^;]*;//' \ + | "${HOME}/FlameGraph/flamegraph.pl" --title "dash-spv sync" --colors hot > "${FLAME_SVG}" + echo "==> wrote ${FLAME_SVG}" +fi diff --git a/dash-spv-bench/scenario.example.yml b/dash-spv-bench/scenario.example.yml new file mode 100644 index 000000000..e15ee3b81 --- /dev/null +++ b/dash-spv-bench/scenario.example.yml @@ -0,0 +1,55 @@ +# dash-spv-bench scenario — every configurable field, documented. +# +# A scenario is ONE self-contained YAML file (its name is the scenario name). Run it with: +# ./run.sh scenarios/ideal-3-peers.yml +# ./run.sh scenarios/ideal-3-peers.yml --flame # profile it -> profiles/flamegraph.svg +# or run the whole scenarios/ directory and get a Markdown report: +# ./bench-scenarios.sh +# +# There is NO .env. Everything below lives in the scenario. Local mode needs only docker (it +# builds the snapshot and the peers in containers); testnet mode needs nothing extra. + +# --------------------------------------------------------------------------------------------- +# COMMON (both modes) +# --------------------------------------------------------------------------------------------- +description: "One-line summary of this scenario" # optional; shown in the report table. + +mode: local # "local" (docker peers you control), "testnet" or "mainnet" (real network). Default: local. + +cpus: "0-3" # optional. taskset list pinning the MEASURED client (here 4 cores: 2,3,4,5). + # In local mode the docker peers get every OTHER core. Omit => no pinning. + +max_peers: 3 # optional. Max simultaneous peers. Omit => the ClientConfig default. + +mnemonic: "job flower agree lyrics industry note boost finger buddy dog exact fat" + # optional. BIP39 phrase whose filter matches drive block download. + # Omit to use the binary's built-in default wallet. + +# --------------------------------------------------------------------------------------------- +# LOCAL mode (mode: local) — docker dashd peers serving the frozen ./chain-data snapshot +# --------------------------------------------------------------------------------------------- +blocks: 1000000 # optional (default 1000000). The target HEIGHT: snapshot-chain.sh builds + # ./chain-data up to here (via docker) and the client syncs the full + # range genesis..blocks. Lower it for a shorter run. + +# Peer groups: one entry per group; each becomes `count` dashd containers sharing a connection +# quality (via `tc netem`). List several groups to mix good and bad peers in one network. +peers: + - count: 3 # required: how many peers in this group + latency_ms: 50 # optional: added one-way delay (netem delay) + jitter_ms: 0 # optional: delay jitter (netem delay ) + loss_pct: 0 # optional: packet loss % (netem loss) + rate_kbit: 0 # optional: bandwidth cap in kbit (netem rate) + corrupt_pct: 0 # optional: bit corruption % (netem corrupt) + reorder_pct: 0 # optional: packet reordering % (netem reorder) + # A second, degraded group ("bad peers"): + # - { count: 1, latency_ms: 800, loss_pct: 8, rate_kbit: 500 } + +# --------------------------------------------------------------------------------------------- +# TESTNET / MAINNET mode — connect to the real Dash network instead of docker peers. +# Set `mode: testnet` or `mode: mainnet` above, then use these instead of blocks/peers: +# --------------------------------------------------------------------------------------------- +# peers: # explicit nodes as "host:port" strings; omit for DNS discovery. +# - "1.2.3.4:9999" # (testnet uses :19999, mainnet :9999) +# - "5.6.7.8:9999" +# start_height: 2000000 # optional checkpoint: skip genesis..start and sync a bounded range. diff --git a/dash-spv-bench/scenarios/congested-2core.yml b/dash-spv-bench/scenarios/congested-2core.yml new file mode 100644 index 000000000..bbb6455d0 --- /dev/null +++ b/dash-spv-bench/scenarios/congested-2core.yml @@ -0,0 +1,5 @@ +description: "CPU-starved client (2 cores), 5 peers on a jittery 300ms link." +cpus: "0-1" +repeats: 3 +peers: + - { count: 5, latency_ms: 300, jitter_ms: 100 } diff --git a/dash-spv-bench/scenarios/ideal-1-peer.yml b/dash-spv-bench/scenarios/ideal-1-peer.yml new file mode 100644 index 000000000..10b9c68ce --- /dev/null +++ b/dash-spv-bench/scenarios/ideal-1-peer.yml @@ -0,0 +1,5 @@ +description: "Ideal link, single peer: 4 CPUs, 1 peer @ 50ms, full 1M sync." +cpus: "0-3" +repeats: 3 +peers: + - { count: 1, latency_ms: 50 } diff --git a/dash-spv-bench/scenarios/ideal-3-peers.yml b/dash-spv-bench/scenarios/ideal-3-peers.yml new file mode 100644 index 000000000..2ba28c255 --- /dev/null +++ b/dash-spv-bench/scenarios/ideal-3-peers.yml @@ -0,0 +1,5 @@ +description: "Ideal link: 4 CPUs, 3 peers @ 50ms, full 1M sync." +cpus: "0-3" +repeats: 3 +peers: + - { count: 3, latency_ms: 50 } diff --git a/dash-spv-bench/scenarios/ideal-5-peers.yml b/dash-spv-bench/scenarios/ideal-5-peers.yml new file mode 100644 index 000000000..95e299642 --- /dev/null +++ b/dash-spv-bench/scenarios/ideal-5-peers.yml @@ -0,0 +1,5 @@ +description: "Ideal link, more peers: 4 CPUs, 5 peers @ 50ms, full 1M sync." +cpus: "0-3" +repeats: 3 +peers: + - { count: 5, latency_ms: 50 } diff --git a/dash-spv-bench/scenarios/no-lag-ideal-1-peer.yml b/dash-spv-bench/scenarios/no-lag-ideal-1-peer.yml new file mode 100644 index 000000000..237e2382b --- /dev/null +++ b/dash-spv-bench/scenarios/no-lag-ideal-1-peer.yml @@ -0,0 +1,5 @@ +description: "No-lag ideal, single peer: 12 CPUs, 1 peer @ 0ms, full 1M sync." +cpus: "0-11" +repeats: 3 +peers: + - { count: 1, latency_ms: 0 } diff --git a/dash-spv-bench/scenarios/no-lag-ideal-3-peers.yml b/dash-spv-bench/scenarios/no-lag-ideal-3-peers.yml new file mode 100644 index 000000000..8f4efdced --- /dev/null +++ b/dash-spv-bench/scenarios/no-lag-ideal-3-peers.yml @@ -0,0 +1,5 @@ +description: "No-lag ideal: 12 CPUs, 3 peers @ 0ms, full 1M sync." +cpus: "0-11" +repeats: 3 +peers: + - { count: 3, latency_ms: 0 } diff --git a/dash-spv-bench/scenarios/no-lag-ideal-5-peers.yml b/dash-spv-bench/scenarios/no-lag-ideal-5-peers.yml new file mode 100644 index 000000000..a0d7dda01 --- /dev/null +++ b/dash-spv-bench/scenarios/no-lag-ideal-5-peers.yml @@ -0,0 +1,5 @@ +description: "No-lag ideal, more peers: 12 CPUs, 5 peers @ 0ms, full 1M sync." +cpus: "0-11" +repeats: 3 +peers: + - { count: 5, latency_ms: 0 } diff --git a/dash-spv-bench/scenarios/one-bad-peer.yml b/dash-spv-bench/scenarios/one-bad-peer.yml new file mode 100644 index 000000000..3eb8187d6 --- /dev/null +++ b/dash-spv-bench/scenarios/one-bad-peer.yml @@ -0,0 +1,6 @@ +description: "Two good peers plus one slow/lossy/capped peer (a bad peer in the set)." +cpus: "0-3" +repeats: 3 +peers: + - { count: 2, latency_ms: 50 } + - { count: 1, latency_ms: 800, loss_pct: 8, rate_kbit: 500 } diff --git a/dash-spv-bench/scenarios/real-mainnet.yml b/dash-spv-bench/scenarios/real-mainnet.yml new file mode 100644 index 000000000..1d053541e --- /dev/null +++ b/dash-spv-bench/scenarios/real-mainnet.yml @@ -0,0 +1,4 @@ +description: "Mainnet sync from a 2M checkpoint via DNS peer discovery." +cpus: "0-11" +max_peers: 3 +mode: mainnet diff --git a/dash-spv-bench/scenarios/real-testnet.yml b/dash-spv-bench/scenarios/real-testnet.yml new file mode 100644 index 000000000..b3435857f --- /dev/null +++ b/dash-spv-bench/scenarios/real-testnet.yml @@ -0,0 +1,4 @@ +description: "Testnet sync from a 500k checkpoint via DNS peer discovery." +cpus: "0-11" +max_peers: 3 +mode: testnet diff --git a/dash-spv-bench/snapshot-chain.sh b/dash-spv-bench/snapshot-chain.sh new file mode 100755 index 000000000..2b312fd52 --- /dev/null +++ b/dash-spv-bench/snapshot-chain.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Extend ./chain-data to BENCH_HEIGHT (downloads only the missing blocks) using the dashd that +# ships INSIDE the peer docker image — so no host dashd / DASHD_PATH / setup-dashd.py is needed, +# only docker. BENCH_HEIGHT comes from the environment (run.sh exports it from the scenario's +# `blocks`). There is no .env. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHAIN_DIR="${SCRIPT_DIR}/chain-data" +SUBDIR="testnet3" +IMAGE="dash-spv-bench/dashd:latest" + +cd "${SCRIPT_DIR}" + +: "${BENCH_HEIGHT:?BENCH_HEIGHT must be set — run.sh exports it from the scenario blocks}" +command -v docker >/dev/null 2>&1 || { echo "Error: docker is required to build the snapshot." >&2; exit 1; } + +# Same image the peers use; build it if we don't have it yet. +docker image inspect "${IMAGE}" >/dev/null 2>&1 \ + || { echo "==> building peer image"; docker build -t "${IMAGE}" -f "${SCRIPT_DIR}/Dockerfile" "${SCRIPT_DIR}"; } + +mkdir -p "${CHAIN_DIR}" + +echo "==> extending snapshot to height ${BENCH_HEIGHT} via docker (downloads only the missing blocks)..." + +# One-shot container: dashd syncs testnet to -stopatheight then exits. Idempotent — if ./chain-data +# is already at (or past) the height it stops right away. --user makes dashd write ./chain-data as +# the host user (not root), so the later CoW clone can read it. +docker run --rm --user "$(id -u):$(id -g)" -v "${CHAIN_DIR}:/data" "${IMAGE}" \ + dashd -testnet -datadir=/data -daemon=0 -server=1 \ + -blockfilterindex=1 -peerblockfilters=1 -stopatheight="${BENCH_HEIGHT}" \ + -txindex=0 -prune=0 -disablewallet=1 + +echo "==> done. Snapshot is at ${CHAIN_DIR}/${SUBDIR} (height ${BENCH_HEIGHT})." diff --git a/dash-spv-bench/src/main.rs b/dash-spv-bench/src/main.rs new file mode 100644 index 000000000..dd2e2de6b --- /dev/null +++ b/dash-spv-bench/src/main.rs @@ -0,0 +1,168 @@ +mod metrics; + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{anyhow, Context, Result}; +use dash_spv::network::PeerNetworkManager; +use dash_spv::storage::DiskStorageManager; +use dash_spv::{ClientConfig, DashSpvClient, EventHandler}; +use dashcore::Network; +use key_wallet::wallet::initialization::WalletAccountCreationOptions; +use key_wallet::wallet::ManagedWalletInfo; +use key_wallet_manager::WalletManager; +use std::collections::BTreeSet; +use tokio::sync::RwLock; + +use crate::metrics::BenchEventHandler; + +/// Default wallet: a mnemonic with real testnet history +const DEFAULT_MNEMONIC: &str = + "job flower agree lyrics industry note boost finger buddy dog exact fat"; + +fn env_or(key: &str, default: &str) -> String { + std::env::var(key).unwrap_or_else(|_| default.to_string()) +} + +#[tokio::main] +async fn main() -> Result<()> { + // Anchor the timeline at process start so `tlog!` timestamps read as time-since-launch. + dash_spv::timer::init(); + + tracing_subscriber::fmt() + // Logs go to stderr so they stream live and stay separate from the + // machine-readable result summary on stdout (which run.sh captures/parses). + // Set RUST_LOG to raise verbosity, e.g. + // RUST_LOG="warn,dash_spv::network=info" ./run.sh scenarios/.yml + .with_writer(std::io::stderr) + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + // Note: `dash_spv=warn` would also silence `dash_spv_bench` (prefix match), so + // name the bench target explicitly. Timeline marks are opt-in — quiet by default; + // enable with e.g. RUST_LOG="warn,dash_spv_bench=info,timeline=info". + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn,dash_spv_bench=info")), + ) + .init(); + + // The mode picks the network. `local` runs against docker peers serving a testnet + // snapshot; `testnet`/`mainnet` sync the real network (DNS discovery or explicit peers). + let mode = env_or("BENCH_MODE", "local").trim().to_ascii_lowercase(); + let (network, remote) = match mode.as_str() { + "local" => (Network::Testnet, false), + "testnet" => (Network::Testnet, true), + "mainnet" => (Network::Mainnet, true), + other => { + return Err(anyhow!( + "BENCH_MODE must be 'local', 'testnet' or 'mainnet', got '{other}'" + )) + } + }; + + // BENCH_PEERS: comma-separated host:port list. Empty => DNS discovery, which is only + // possible on testnet/mainnet — local docker peers can't be discovered, so local requires them. + let peers: Vec = std::env::var("BENCH_PEERS") + .unwrap_or_default() + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|p| p.parse::().with_context(|| format!("bad peer {p}"))) + .collect::>()?; + + if !remote && peers.is_empty() { + return Err(anyhow!("BENCH_MODE=local requires BENCH_PEERS (the docker peer addresses)")); + } + + let dns_mode = remote && peers.is_empty(); + + // Optional checkpoint start height: skip genesis..start and sync a bounded range. + let start_height: Option = + std::env::var("BENCH_START_HEIGHT").ok().and_then(|v| v.trim().parse().ok()); + + let mnemonic = env_or("BENCH_MNEMONIC", DEFAULT_MNEMONIC); + let timeout = Duration::from_secs(1800); + + tracing::info!( + "dash-spv-bench: mode={} ({}) on {:?}, wallet={}", + mode, + if dns_mode { + "DNS discovery".to_string() + } else { + format!("{} peers", peers.len()) + }, + network, + !mnemonic.is_empty(), + ); + + // Fresh storage dir => cold cache. + let storage_dir = tempfile::tempdir().context("temp storage dir")?; + let mut config = ClientConfig::new(network).with_storage_path(storage_dir.path().to_path_buf()); + config.enable_filters = true; + config.enable_masternodes = false; + config.enable_mempool_tracking = false; + config.start_from_height = start_height; + config.restrict_to_configured_peers = !dns_mode; + // `max_peers` only if the scenario set it; otherwise keep the ClientConfig default. + if let Ok(v) = std::env::var("BENCH_MAX_PEERS") { + if let Ok(n) = v.trim().parse() { + config.max_peers = n; + } + } + for addr in &peers { + config.add_peer(*addr); + } + + let network_manager = + PeerNetworkManager::new(&config).await.map_err(|e| anyhow!("network manager: {e}"))?; + let storage_manager = + DiskStorageManager::new(&config).await.map_err(|e| anyhow!("storage manager: {e}"))?; + + // Wallet: a single BIP44 account 0 from the mnemonic, so filter matches drive block download. + let mut wallet_manager = WalletManager::::new(network); + if !mnemonic.is_empty() { + wallet_manager + .create_wallet_from_mnemonic(&mnemonic, 0, account_creation_options()) + .map_err(|e| anyhow!("wallet from mnemonic: {e}"))?; + } + let wallet = Arc::new(RwLock::new(wallet_manager)); + + let handler = Arc::new(BenchEventHandler::new()); + let client = DashSpvClient::new( + config, + network_manager, + storage_manager, + wallet, + vec![handler.clone() as Arc], + ) + .await + .map_err(|e| anyhow!("client new: {e}"))?; + + // Run the client on its own task; stop as soon as the sync completes (or times out). + let run_client = client.clone(); + let run_handle = tokio::spawn(async move { + let _ = run_client.run().await; + }); + + let _ = tokio::time::timeout(timeout, handler.wait_done()).await; + let m = handler.snapshot(); + dash_spv::timer::dump_profile(); + + let _ = client.shutdown().await; + run_handle.abort(); + let _ = run_handle.await; + drop(storage_dir); + + println!("\n{m}"); + Ok(()) +} + +fn account_creation_options() -> WalletAccountCreationOptions { + WalletAccountCreationOptions::SpecificAccounts( + BTreeSet::from([0]), + BTreeSet::new(), + BTreeSet::new(), + BTreeSet::new(), + BTreeSet::new(), + None, + ) +} diff --git a/dash-spv-bench/src/metrics.rs b/dash-spv-bench/src/metrics.rs new file mode 100644 index 000000000..328880dad --- /dev/null +++ b/dash-spv-bench/src/metrics.rs @@ -0,0 +1,161 @@ +use std::collections::BTreeMap; +use std::fmt; +use std::sync::Mutex; +use std::time::Instant; + +use dash_spv::sync::SyncEvent; +use dash_spv::EventHandler; +use tokio::sync::Notify; + +/// Milestones we time, in pipeline order. Stable string keys so timings are easy to correlate. +pub const MILESTONE_BLOCK_HEADERS: &str = "block_headers_complete"; +pub const MILESTONE_FILTER_HEADERS: &str = "filter_headers_complete"; +pub const MILESTONE_FILTERS: &str = "filters_complete"; +pub const MILESTONE_SYNC: &str = "sync_complete"; + +#[derive(Debug)] +pub struct RunMetrics { + /// Total wall-clock from start to `sync_complete` (or to abort/timeout). + total_ms: u64, + /// Whether the run reached `sync_complete` within the timeout. If false, the run timed out + completed: bool, + block_headers_ms: Option, + filter_headers_ms: Option, + filters_ms: Option, + /// How long after block-header completion the filter headers finished + filter_headers_lag_ms: Option, + error: Option, +} + +impl fmt::Display for RunMetrics { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let ms = |o: Option| o.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()); + + writeln!(f, "=== dash-spv sync ===")?; + writeln!( + f, + "completed: {}{}", + self.completed, + if self.completed { + "" + } else { + " (TIMED OUT)" + } + )?; + writeln!(f, "total_ms: {}", self.total_ms)?; + writeln!(f, "block_headers_ms: {}", ms(self.block_headers_ms))?; + writeln!(f, "filter_headers_ms: {}", ms(self.filter_headers_ms))?; + writeln!(f, "filters_ms: {}", ms(self.filters_ms))?; + write!(f, "filter_hdr_lag_ms: {}", ms(self.filter_headers_lag_ms))?; + if let Some(e) = &self.error { + write!(f, "\nerror: {e}")?; + } + + Ok(()) + } +} + +/// Collects timing for one run by subscribing to the client's event stream. +pub struct BenchEventHandler { + start: Instant, + inner: Mutex, + /// Notified once the run reaches `sync_complete`. + done: Notify, +} + +struct Inner { + /// label -> elapsed_ms at first occurrence. + milestones: BTreeMap<&'static str, u64>, + error: Option, + completed: bool, +} + +impl BenchEventHandler { + pub fn new() -> Self { + Self { + start: Instant::now(), + inner: Mutex::new(Inner { + milestones: BTreeMap::new(), + error: None, + completed: false, + }), + done: Notify::new(), + } + } + + fn record(&self, label: &'static str) { + let elapsed = self.start.elapsed().as_millis() as u64; + let mut inner = self.inner.lock().unwrap(); + inner.milestones.entry(label).or_insert(elapsed); + } + + /// Wait until the run signals completion. Returns immediately if already done. + pub async fn wait_done(&self) { + if self.inner.lock().unwrap().completed { + return; + } + self.done.notified().await; + } + + /// Snapshot the collected timing into a [`RunMetrics`]. `total_ms` is the elapsed time + /// at the moment of snapshotting (call after `wait_done` or timeout). + pub fn snapshot(&self) -> RunMetrics { + let inner = self.inner.lock().unwrap(); + let bh = inner.milestones.get(MILESTONE_BLOCK_HEADERS).copied(); + let fh = inner.milestones.get(MILESTONE_FILTER_HEADERS).copied(); + let fl = inner.milestones.get(MILESTONE_FILTERS).copied(); + let sc = inner.milestones.get(MILESTONE_SYNC).copied(); + + RunMetrics { + total_ms: sc.unwrap_or_else(|| self.start.elapsed().as_millis() as u64), + completed: inner.completed, + block_headers_ms: bh, + filter_headers_ms: fh, + filters_ms: fl, + filter_headers_lag_ms: match (bh, fh) { + (Some(b), Some(f)) => Some(f.saturating_sub(b)), + _ => None, + }, + error: inner.error.clone(), + } + } +} + +impl Default for BenchEventHandler { + fn default() -> Self { + Self::new() + } +} + +impl EventHandler for BenchEventHandler { + fn on_sync_event(&self, event: &SyncEvent) { + match event { + SyncEvent::BlockHeaderSyncComplete { + .. + } => self.record(MILESTONE_BLOCK_HEADERS), + SyncEvent::FilterHeadersSyncComplete { + .. + } => self.record(MILESTONE_FILTER_HEADERS), + SyncEvent::FiltersSyncComplete { + .. + } => self.record(MILESTONE_FILTERS), + SyncEvent::SyncComplete { + .. + } => { + self.record(MILESTONE_SYNC); + let mut inner = self.inner.lock().unwrap(); + inner.completed = true; + drop(inner); + self.done.notify_waiters(); + } + _ => {} + } + } + + fn on_error(&self, error: &str) { + let mut inner = self.inner.lock().unwrap(); + if inner.error.is_none() { + inner.error = Some(error.to_string()); + } + } +} diff --git a/dash-spv/src/lib.rs b/dash-spv/src/lib.rs index ecf8f432a..bf8d099f2 100644 --- a/dash-spv/src/lib.rs +++ b/dash-spv/src/lib.rs @@ -69,6 +69,7 @@ pub mod logging; pub mod network; pub mod storage; pub mod sync; +pub mod timer; pub mod types; pub mod validation; diff --git a/dash-spv/src/timer.rs b/dash-spv/src/timer.rs new file mode 100644 index 000000000..bf7c8cc0f --- /dev/null +++ b/dash-spv/src/timer.rs @@ -0,0 +1,148 @@ +//! Process-relative timeline timestamps for pinpointing where sync wall-clock time goes. +//! +//! One process-start [`Instant`] plus the [`tlog!`](crate::tlog) macro: drop `tlog!("...")` at +//! interesting points (before/after a network round-trip, before/after hashing, storage writes, …) +//! and every line is prefixed with the elapsed time since the timeline was anchored. The gap +//! between two consecutive timeline lines is the time spent in between — so you can add marks +//! incrementally and bisect where the time is going. +//! +//! # Usage +//! ```ignore +//! dash_spv::timer::init(); // once, at the top of main() (optional; otherwise anchored lazily) +//! tlog!("about to send GetHeaders (seg {})", seg_id); +//! // ... network round-trip ... +//! tlog!("got {} headers", headers.len()); +//! ``` +//! Timeline lines use the `timeline` tracing target at INFO — enable them with `timeline=info` +//! in `RUST_LOG` / the subscriber's env filter. + +use std::sync::LazyLock; +use std::time::{Duration, Instant}; + +/// The instant the timeline started. Anchored on first access; call [`init`] early in `main` to +/// anchor it at process start rather than at the first [`elapsed`]/`tlog!`. +pub static START: LazyLock = LazyLock::new(Instant::now); + +/// Anchor the timeline now (idempotent). Call once at the top of `main` so `t+0` == process start. +pub fn init() { + LazyLock::force(&START); +} + +/// Time elapsed since the timeline started. +#[inline] +pub fn elapsed() -> Duration { + START.elapsed() +} + +/// Milliseconds elapsed since the timeline started (what [`tlog!`](crate::tlog) prints). +#[inline] +pub fn elapsed_ms() -> u128 { + START.elapsed().as_millis() +} + +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; + +/// A named time accumulator for micro-profiling a hot path called too often to `tlog!` per call +/// (e.g. once per cfilter message). Sum time with [`Prof::add`], print all with [`dump_profile`]. +pub struct Prof { + pub name: &'static str, + ns: AtomicU64, + calls: AtomicU64, +} + +impl Prof { + pub const fn new(name: &'static str) -> Self { + Self { + name, + ns: AtomicU64::new(0), + calls: AtomicU64::new(0), + } + } + /// Add one sample. + #[inline] + pub fn add(&self, d: Duration) { + self.ns.fetch_add(d.as_nanos() as u64, Relaxed); + self.calls.fetch_add(1, Relaxed); + } +} + +// Buckets for the per-cfilter-message hot path (filters/sync_manager.rs::handle_message). +pub static P_HEIGHT_LOOKUP: Prof = Prof::new("filt.height_lookup"); +pub static P_RECV_DATA: Prof = Prof::new("filt.receive_with_data"); +pub static P_SEND_PENDING: Prof = Prof::new("filt.send_pending"); +pub static P_STORE_MATCH: Prof = Prof::new("filt.store_and_match"); + +// Arrival path: time to lock the shared Arc> and dispatch each message. +// If per-call time balloons with more peers, that Mutex is the ~70k/s serialization point. +pub static P_NET_DISPATCH: Prof = Prof::new("net.dispatch(lock)"); + +// Reader loop (network/manager.rs) per-iteration breakdown. RLOCK+WLOCK are the two peer-lock +// acquisitions per message; RECV is the receive_message/select; MSG vs IDLE count how often an +// iteration yielded a message vs timed out waiting (IDLE-heavy => peer/network-bound, not reader). +pub static P_READ_RLOCK: Prof = Prof::new("read.rlock"); +pub static P_READ_WLOCK: Prof = Prof::new("read.wlock"); +pub static P_READ_RECV: Prof = Prof::new("read.recv/select"); +pub static P_READ_MSG: Prof = Prof::new("read.got_msg"); +pub static P_READ_IDLE: Prof = Prof::new("read.idle_timeout"); + +// Lock contention probes for the cfilter phase. +// READER_HOLD: how long the reader holds peer.write() across receive_message/select (if long, it +// blocks the sender which needs the same lock). SENDER_WAIT: how long send_message_to_peer waits +// to acquire peer.write() (high => reader is hogging it). HM_HDR: per-cfilter header_storage.read +// acquire wait in handle_message. DISPATCH_WAIT: message_dispatcher Mutex acquire wait. +pub static P_READER_HOLD: Prof = Prof::new("reader.peer_write_HOLD"); +pub static P_SENDER_WAIT: Prof = Prof::new("sender.peer_write_WAIT"); +pub static P_HM_HDR: Prof = Prof::new("handle_msg.hdr_read_wait"); +pub static P_DISPATCH_WAIT: Prof = Prof::new("dispatch.lock_WAIT"); + +/// Log all profiling accumulators (call once at end of run). Uses the `timeline` target. +pub fn dump_profile() { + // eprintln so it prints even when the `timeline` target is filtered out (so we can profile with + // the high-volume per-message marks disabled). + for p in [ + &P_HEIGHT_LOOKUP, + &P_RECV_DATA, + &P_SEND_PENDING, + &P_STORE_MATCH, + &P_NET_DISPATCH, + &P_READ_RLOCK, + &P_READ_WLOCK, + &P_READ_RECV, + &P_READ_MSG, + &P_READ_IDLE, + &P_READER_HOLD, + &P_SENDER_WAIT, + &P_HM_HDR, + &P_DISPATCH_WAIT, + ] { + let ns = p.ns.load(Relaxed); + let c = p.calls.load(Relaxed); + if c > 0 { + eprintln!( + "[t+{:>7}ms] PROF {:22} calls={:>8} total={:>6}ms per={:>7.3}us", + elapsed_ms(), + p.name, + c, + ns / 1_000_000, + ns as f64 / c as f64 / 1000.0 + ); + } + } +} + +/// Log a message prefixed with the elapsed time since the process timeline started, e.g. +/// `[t+ 1234ms] sent GetHeaders`. Uses the `timeline` tracing target at INFO level. +/// +/// Cheap to leave in place, easy to add more of — the point is to sprinkle these around hot spots +/// and read off the gaps between consecutive lines. +#[macro_export] +macro_rules! tlog { + ($($arg:tt)*) => { + ::tracing::info!( + target: "timeline", + "[t+{:>7}ms] {}", + $crate::timer::elapsed_ms(), + format_args!($($arg)*) + ) + }; +} From d08aae33c0857f89be13a65b7ed9801b570d86aa Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Sat, 11 Jul 2026 08:18:50 +0000 Subject: [PATCH 02/25] refactor(dash-spv): refactor network module --- dash-spv/Cargo.toml | 2 +- dash-spv/src/client/core.rs | 23 +- dash-spv/src/client/event_handler.rs | 20 +- dash-spv/src/client/events.rs | 6 +- dash-spv/src/client/lifecycle.rs | 21 +- dash-spv/src/client/mod.rs | 97 - dash-spv/src/client/queries.rs | 15 +- dash-spv/src/client/sync_coordinator.rs | 3 +- dash-spv/src/client/transactions.rs | 18 +- dash-spv/src/lib.rs | 1 + dash-spv/src/network2/discovery.rs | 71 + dash-spv/src/network2/manager.rs | 1036 ++++++ dash-spv/src/network2/mod.rs | 5 + dash-spv/src/network2/peer.rs | 536 +++ dash-spv/src/storage/segments.rs | 276 +- dash-spv/src/sync/block_headers/manager.rs | 363 +- dash-spv/src/sync/block_headers/pipeline.rs | 353 +- .../src/sync/block_headers/segment_state.rs | 320 +- .../src/sync/block_headers/sync_manager.rs | 126 +- dash-spv/src/sync/blocks/manager.rs | 225 +- dash-spv/src/sync/blocks/pipeline.rs | 577 +--- dash-spv/src/sync/blocks/sync_manager.rs | 80 +- dash-spv/src/sync/chainlock/manager.rs | 357 -- dash-spv/src/sync/chainlock/sync_manager.rs | 23 +- dash-spv/src/sync/download_coordinator.rs | 402 +-- dash-spv/src/sync/events.rs | 31 + dash-spv/src/sync/filter_headers/manager.rs | 251 +- dash-spv/src/sync/filter_headers/pipeline.rs | 325 +- dash-spv/src/sync/filter_headers/progress.rs | 59 +- .../src/sync/filter_headers/sync_manager.rs | 79 +- dash-spv/src/sync/filters/batch.rs | 59 +- dash-spv/src/sync/filters/batch_tracker.rs | 5 + dash-spv/src/sync/filters/manager.rs | 3015 +++++------------ dash-spv/src/sync/filters/pipeline.rs | 1000 +----- dash-spv/src/sync/filters/progress.rs | 19 +- dash-spv/src/sync/filters/sync_manager.rs | 96 +- dash-spv/src/sync/instantsend/manager.rs | 176 - dash-spv/src/sync/instantsend/sync_manager.rs | 22 +- dash-spv/src/sync/masternodes/manager.rs | 424 +-- dash-spv/src/sync/masternodes/pipeline.rs | 280 +- dash-spv/src/sync/masternodes/sync_manager.rs | 395 +-- dash-spv/src/sync/mempool/manager.rs | 1215 +------ dash-spv/src/sync/mempool/sync_manager.rs | 722 +--- dash-spv/src/sync/mod.rs | 2 +- dash-spv/src/sync/sync_coordinator.rs | 32 +- dash-spv/src/sync/sync_manager.rs | 257 +- dash-spv/src/test_utils/event_handler.rs | 2 +- dash-spv/src/timer.rs | 34 + dash-spv/src/validation/filter.rs | 4 +- dash-spv/src/validation/header.rs | 7 +- dash/Cargo.toml | 3 + dash/src/network/message.rs | 132 + 52 files changed, 4165 insertions(+), 9437 deletions(-) create mode 100644 dash-spv/src/network2/discovery.rs create mode 100644 dash-spv/src/network2/manager.rs create mode 100644 dash-spv/src/network2/mod.rs create mode 100644 dash-spv/src/network2/peer.rs diff --git a/dash-spv/Cargo.toml b/dash-spv/Cargo.toml index 0c80f5b36..c8b470944 100644 --- a/dash-spv/Cargo.toml +++ b/dash-spv/Cargo.toml @@ -10,7 +10,7 @@ rust-version = "1.89" [dependencies] # Core Dash libraries -dashcore = { path = "../dash", features = ["serde", "core-block-hash-use-x11", "message_verification", "bls", "quorum_validation"] } +dashcore = { path = "../dash", features = ["serde", "core-block-hash-use-x11", "message_verification", "bls", "quorum_validation", "tokio"] } dashcore_hashes = { path = "../hashes" } dash-network-seeds = { path = "../dash-network-seeds" } key-wallet = { path = "../key-wallet" } diff --git a/dash-spv/src/client/core.rs b/dash-spv/src/client/core.rs index 09e8bb712..c8c1b714b 100644 --- a/dash-spv/src/client/core.rs +++ b/dash-spv/src/client/core.rs @@ -14,7 +14,6 @@ use tokio::sync::{watch, Mutex, RwLock}; use super::ClientConfig; use crate::error::{Result, SpvError}; -use crate::network::NetworkManager; use crate::storage::{ PersistentBlockHeaderStorage, PersistentBlockStorage, PersistentFilterHeaderStorage, PersistentFilterStorage, PersistentMetadataStorage, StorageManager, @@ -35,8 +34,8 @@ pub(super) type PersistentSyncCoordinator = SyncCoordinator< /// /// # Generic Design Philosophy /// -/// This struct uses three generic parameters (`W`, `N`, `S`) instead of concrete types or -/// trait objects. This design choice provides significant benefits for a library: +/// This struct uses two generic parameters (`W`, `S`) instead of concrete types or +/// trait objects. This design choice provides significant benefits for a library. /// /// ## Benefits of Generic Architecture /// @@ -70,8 +69,8 @@ pub(super) type PersistentSyncCoordinator = SyncCoordinator< /// ## Type Parameters /// /// - `W: WalletInterface` - Handles UTXO tracking, address management, transaction processing -/// - `N: NetworkManager` - Manages peer connections, message routing, network protocol /// - `S: StorageManager` - Persistent storage for headers, filters, chain state +/// - Networking is the concrete `crate::network2::PeerNetworkManager` /// - Event handlers are stored as `Vec>` /// /// ## Common Configurations @@ -82,14 +81,6 @@ pub(super) type PersistentSyncCoordinator = SyncCoordinator< /// // Production configuration /// type StandardSpvClient = DashSpvClient< /// WalletManager, -/// PeerNetworkManager, -/// DiskStorageManager, -/// >; -/// -/// // Test configuration -/// type TestSpvClient = DashSpvClient< -/// WalletManager, -/// MockNetworkManager, /// DiskStorageManager, /// >; /// ``` @@ -103,9 +94,9 @@ pub(super) type PersistentSyncCoordinator = SyncCoordinator< /// - Not reduce binary size (production has one instantiation anyway) /// /// The generic design is an intentional, beneficial architectural choice for a library. -pub struct DashSpvClient { +pub struct DashSpvClient { pub(super) config: Arc>, - pub(super) network: Arc>, + pub(super) network: Arc, pub(super) storage: Arc>, /// External wallet implementation (required) pub(super) wallet: Arc>, @@ -117,7 +108,7 @@ pub struct DashSpvClient>>, } -impl Clone for DashSpvClient { +impl Clone for DashSpvClient { fn clone(&self) -> Self { Self { config: Arc::clone(&self.config), @@ -132,7 +123,7 @@ impl Clone for DashSpv } } -impl DashSpvClient { +impl DashSpvClient { // ============ Simple Getters ============ /// Get a reference to the wallet. diff --git a/dash-spv/src/client/event_handler.rs b/dash-spv/src/client/event_handler.rs index a6ef1aace..bc5264e6a 100644 --- a/dash-spv/src/client/event_handler.rs +++ b/dash-spv/src/client/event_handler.rs @@ -11,7 +11,7 @@ use tokio::sync::{broadcast, mpsc, watch, RwLock}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; -use crate::network::NetworkEvent; +use crate::network2::NetworkEvent; use crate::sync::{SyncEvent, SyncProgress}; use dashcore::ephemerealdata::chain_lock::ChainLock; use key_wallet_manager::{WalletEvent, WalletInterface}; @@ -227,7 +227,7 @@ mod tests { use tokio_util::sync::CancellationToken; use super::{spawn_broadcast_monitor, spawn_progress_monitor, EventHandler}; - use crate::network::NetworkEvent; + use crate::network2::NetworkEvent; use crate::sync::{ManagerIdentifier, SyncEvent, SyncProgress}; use key_wallet_manager::WalletEvent; @@ -282,11 +282,7 @@ mod tests { tip_height: 100, }; handler.on_sync_event(&event); - handler.on_network_event(&NetworkEvent::PeersUpdated { - connected_count: 0, - addresses: vec![], - best_height: None, - }); + handler.on_network_event(&NetworkEvent::PeersUpdated); handler.on_progress(&SyncProgress::default()); handler.on_error("test error"); } @@ -490,14 +486,8 @@ mod tests { ); let addr: SocketAddr = "127.0.0.1:9999".parse().unwrap(); - tx.send(NetworkEvent::PeerConnected { - address: addr, - }) - .unwrap(); - tx.send(NetworkEvent::PeerDisconnected { - address: addr, - }) - .unwrap(); + tx.send(NetworkEvent::PeerConnected(addr)).unwrap(); + tx.send(NetworkEvent::PeerDisconnected(addr)).unwrap(); tokio::time::sleep(std::time::Duration::from_millis(50)).await; shutdown.cancel(); diff --git a/dash-spv/src/client/events.rs b/dash-spv/src/client/events.rs index 6ed109ed9..7da9abb93 100644 --- a/dash-spv/src/client/events.rs +++ b/dash-spv/src/client/events.rs @@ -6,7 +6,7 @@ use tokio::sync::watch; -use crate::network::{NetworkEvent, NetworkManager}; +use crate::network2::NetworkEvent; use crate::storage::StorageManager; use crate::sync::{SyncEvent, SyncProgress}; use key_wallet_manager::WalletInterface; @@ -14,7 +14,7 @@ use tokio::sync::broadcast; use super::DashSpvClient; -impl DashSpvClient { +impl DashSpvClient { /// Subscribe to sync progress updates via watch channel. pub(crate) async fn subscribe_progress(&self) -> watch::Receiver { self.sync_coordinator.lock().await.subscribe_progress() @@ -32,6 +32,6 @@ impl DashSpvClient broadcast::Receiver { - self.network.lock().await.subscribe_network_events() + self.network.events() } } diff --git a/dash-spv/src/client/lifecycle.rs b/dash-spv/src/client/lifecycle.rs index 2f789bdf8..65aea96e1 100644 --- a/dash-spv/src/client/lifecycle.rs +++ b/dash-spv/src/client/lifecycle.rs @@ -11,7 +11,6 @@ use super::{ClientConfig, DashSpvClient, EventHandler}; use crate::chain::checkpoints::CheckpointManager; use crate::error::{Result, SpvError}; -use crate::network::NetworkManager; use crate::storage::{ PersistentBlockHeaderStorage, PersistentBlockStorage, PersistentFilterHeaderStorage, PersistentFilterStorage, PersistentMetadataStorage, StorageManager, @@ -31,11 +30,11 @@ use key_wallet_manager::WalletInterface; use std::sync::Arc; use tokio::sync::{watch, Mutex, RwLock}; -impl DashSpvClient { +impl DashSpvClient { /// Create a new SPV client with the given configuration, network, storage, and wallet. pub async fn new( config: ClientConfig, - network: N, + network: crate::network2::PeerNetworkManager, mut storage: S, wallet: Arc>, event_handlers: Vec>, @@ -151,7 +150,7 @@ impl DashSpvClient DashSpvClient DashSpvClient, - ) -> (u32, Option) { - let mut wallet_manager = WalletManager::::new(Network::Mainnet); - wallet_manager - .create_wallet_from_mnemonic( - TEST_MNEMONIC, - birth_height, - WalletAccountCreationOptions::Default, - ) - .expect("wallet creation must succeed"); - let wallet = Arc::new(RwLock::new(wallet_manager)); - - let temp_dir = TempDir::new().unwrap(); - let mut config = ClientConfig::mainnet() - .without_filters() - .without_masternodes() - .with_storage_path(temp_dir.path()); - config.start_from_height = start_from_height; - - let storage = DiskStorageManager::new(&config).await.expect("Failed to create storage"); - let client = DashSpvClient::new( - config, - MockNetworkManager::new(), - storage, - wallet, - vec![Arc::new(())], - ) - .await - .expect("client construction must succeed"); - (client.tip_height().await, client.tip_hash().await) - } - - #[tokio::test] - async fn birth_height_anchors_chain_to_nearest_checkpoint() { - // A wallet born at 120_000 anchors at the 100_000 checkpoint, not genesis, and - // the stored tip carries the trusted checkpoint hash (what the next header's - // `prev_blockhash` is validated against), not a hash recomputed from a header. - let (height, hash) = anchored_tip(120_000, None).await; - assert_eq!(height, 100_000); - let expected: dashcore::BlockHash = - "00000000000fd08c2fb661d2fcb0d49abb3a91e5f27082ce64feed3b4dede2e2".parse().unwrap(); - assert_eq!(hash, Some(expected)); - - // Birth height 0 keeps syncing from genesis. - assert_eq!(anchored_tip(0, None).await.0, 0); - - // Explicit `start_from_height` wins over the wallet birth height, even when - // the birth height would resolve to a higher checkpoint. - assert_eq!(anchored_tip(120_000, Some(60_000)).await.0, 50_000); - } - - #[tokio::test] - async fn client_exposes_shared_wallet_manager() { - let config = ClientConfig::mainnet() - .without_filters() - .without_masternodes() - .with_mempool_tracking(MempoolStrategy::FetchAll) - .with_storage_path(TempDir::new().unwrap().path()); - - let network_manager = MockNetworkManager::new(); - let storage = - DiskStorageManager::with_temp_dir().await.expect("Failed to create tmp storage"); - let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); - - let client = - DashSpvClient::new(config, network_manager, storage, wallet, vec![Arc::new(())]) - .await - .expect("client construction must succeed"); - - // Verify the wallet is accessible - let wallet_ref = client.wallet(); - let _wallet_guard = wallet_ref.read().await; - } -} diff --git a/dash-spv/src/client/queries.rs b/dash-spv/src/client/queries.rs index 00836b220..c4ecbdddc 100644 --- a/dash-spv/src/client/queries.rs +++ b/dash-spv/src/client/queries.rs @@ -7,7 +7,6 @@ //! - Filter availability checks use crate::error::{Result, SpvError}; -use crate::network::NetworkManager; use crate::storage::StorageManager; use dashcore::sml::llmq_type::LLMQType; use dashcore::sml::masternode_list_engine::MasternodeListEngine; @@ -19,19 +18,7 @@ use tokio::sync::RwLock; use super::DashSpvClient; -impl DashSpvClient { - // ============ Peer Queries ============ - - /// Get the number of connected peers. - pub async fn peer_count(&self) -> usize { - self.network.lock().await.peer_count() - } - - /// Disconnect a specific peer. - pub async fn disconnect_peer(&self, addr: &std::net::SocketAddr, reason: &str) -> Result<()> { - Ok(self.network.lock().await.disconnect_peer(addr, reason).await?) - } - +impl DashSpvClient { // ============ Masternode Queries ============ /// Get a reference to the masternode list engine. diff --git a/dash-spv/src/client/sync_coordinator.rs b/dash-spv/src/client/sync_coordinator.rs index e428bcb80..349fe323e 100644 --- a/dash-spv/src/client/sync_coordinator.rs +++ b/dash-spv/src/client/sync_coordinator.rs @@ -10,7 +10,6 @@ use super::event_handler::{ }; use super::DashSpvClient; use crate::error::Result; -use crate::network::NetworkManager; use crate::storage::StorageManager; use crate::sync::SyncProgress; use crate::SpvError; @@ -18,7 +17,7 @@ use key_wallet_manager::WalletInterface; const SYNC_COORDINATOR_TICK_MS: Duration = Duration::from_millis(100); -impl DashSpvClient { +impl DashSpvClient { /// Get current sync progress. pub async fn sync_progress(&self) -> SyncProgress { self.sync_coordinator.lock().await.progress().clone() diff --git a/dash-spv/src/client/transactions.rs b/dash-spv/src/client/transactions.rs index 78b170810..1ca605659 100644 --- a/dash-spv/src/client/transactions.rs +++ b/dash-spv/src/client/transactions.rs @@ -1,31 +1,23 @@ //! Transaction-related client APIs (e.g., broadcasting) -use crate::error::{NetworkError, Result, SpvError}; -use crate::network::NetworkManager; +use crate::error::Result; use crate::storage::StorageManager; use dashcore::network::message::NetworkMessage; use key_wallet_manager::WalletInterface; use super::DashSpvClient; -impl DashSpvClient { +impl DashSpvClient { /// Broadcast a transaction to all connected peers. /// /// The transaction is also injected into the local message pipeline so that /// the mempool manager processes it immediately. pub async fn broadcast_transaction(&self, tx: &dashcore::Transaction) -> Result<()> { - let network_guard = self.network.lock().await; - - if network_guard.peer_count() == 0 { - return Err(SpvError::Network(NetworkError::NotConnected)); - } - - let message = NetworkMessage::Tx(tx.clone()); - network_guard.broadcast(message).await?; + let message = NetworkMessage::Tx(Box::new(tx.clone())); + self.network.broadcast(message); // Inject locally so the mempool manager picks it up through handle_tx. - network_guard.dispatch_local(NetworkMessage::Tx(tx.clone())).await; - + self.network.dispatch_local(NetworkMessage::Tx(Box::new(tx.clone()))).await; Ok(()) } } diff --git a/dash-spv/src/lib.rs b/dash-spv/src/lib.rs index bf8d099f2..a5cee8c80 100644 --- a/dash-spv/src/lib.rs +++ b/dash-spv/src/lib.rs @@ -67,6 +67,7 @@ pub mod client; pub mod error; pub mod logging; pub mod network; +pub mod network2; pub mod storage; pub mod sync; pub mod timer; diff --git a/dash-spv/src/network2/discovery.rs b/dash-spv/src/network2/discovery.rs new file mode 100644 index 000000000..8d144153b --- /dev/null +++ b/dash-spv/src/network2/discovery.rs @@ -0,0 +1,71 @@ +use std::net::SocketAddr; + +use dashcore::Network; +use rand::seq::SliceRandom; + +use crate::network::discovery::DnsDiscovery; +use crate::network2::peer::DisconnectedPeer; +use crate::ClientConfig; + +pub struct PeerDiscoverer { + network: Network, + // Empty means "use DNS discovery" + fixed: Vec, + dns_discovered: Option>, + dns: DnsDiscovery, +} + +impl PeerDiscoverer { + pub fn new(config: &ClientConfig) -> PeerDiscoverer { + PeerDiscoverer { + network: config.network, + fixed: config.peers.clone(), + dns_discovered: None, + dns: DnsDiscovery::new(), + } + } + + pub async fn get(&mut self, count: usize) -> Vec { + let addrs: Vec = if !self.fixed.is_empty() { + self.fixed.choose_multiple(&mut rand::thread_rng(), count).copied().collect() + } else { + let peers = match self.dns_discovered.as_mut() { + Some(known) => known, + None => { + let mut discovered = Self::dns_discover_peers(self.network).await; + discovered.shuffle(&mut rand::thread_rng()); + self.dns_discovered.insert(discovered) + } + }; + + let take = count.min(peers.len()); + peers.drain(..take).collect() + }; + + addrs.into_iter().map(|addr| DisconnectedPeer::new(addr, self.network)).collect() + } + + async fn dns_discover_peers(network: Network) -> Vec { + let seeds = network.dns_seeds(); + let port = network.default_p2p_port(); + let mut addresses = dash_network_seeds::addresses(network); + + for seed in seeds { + match tokio::net::lookup_host((*seed, port)).await { + Ok(iter) => { + let resolved: Vec = iter.collect(); + tracing::info!("DNS seed {} returned {} addresses", seed, resolved.len()); + addresses.extend(resolved); + } + Err(e) => { + tracing::warn!("Failed to resolve DNS seed {} (backup source): {}", seed, e); + } + } + } + + addresses.sort(); + addresses.dedup(); + + addresses + } +} diff --git a/dash-spv/src/network2/manager.rs b/dash-spv/src/network2/manager.rs new file mode 100644 index 000000000..e378a54f8 --- /dev/null +++ b/dash-spv/src/network2/manager.rs @@ -0,0 +1,1036 @@ +use std::{ + collections::{HashMap, VecDeque}, + net::SocketAddr, + sync::{ + atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +use dashcore::network::message::NetworkMessage; +use dashcore::network::message_blockdata::Inventory; +use futures::future::join_all; +use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; +use tokio::sync::{broadcast, Mutex, Notify}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +const HIGH_DEMAND_QUEUE: usize = 40; +const ADD_PEERS_BATCH: usize = 4; +const PROBE_BATCH: usize = 256; +/// Safety ceiling on how many peers backpressure may auto-connect. `max_peers` +/// (config) is the base/initial set; when the send queue stays backed up the +/// peers can't serve fast enough, so we recruit more — up to this cap — to raise +/// aggregate serving capacity. +/// +/// Kept near the empirical sweet spot: on testnet, throughput improves up to +/// ~16 peers, is flat-to-worse by 24-32, and OUTRIGHT STALLS around 64 — dozens +/// of slow peers each pin their measured in-flight on batches that never finish, +/// so there are no completions to wake the router. More peers past the knee only +/// adds congestion, not download rate (the peers, not our link, are the limit). +const MAX_CONNECTED_PEERS: usize = 16; + +/// How long the router waits on a full-capacity stall before checking whether the +/// "in-flight" requests holding that capacity are actually dead. +const STALL_CHECK: Duration = Duration::from_secs(5); + +/// A request unanswered for this long has its slot reclaimed: the owning pipeline +/// has long since timed it out and re-queued it elsewhere, so the slot holds +/// nothing. +/// +/// Generous, because slow is not dead. Blocks are paced through the same budget, +/// and a 2MB block over a weak link legitimately takes a while — the point is to +/// free capacity, not to punish peers (whether the peer is dropped is decided +/// separately, on whether it has EVER answered). +const STALE_REQUEST: Duration = Duration::from_secs(90); +use crate::{ + network2::{ + discovery::PeerDiscoverer, + peer::{ConnectedPeer, DisconnectedPeer, PeerEvent}, + }, + ClientConfig, +}; + +/// An inbound message on its way to the managers that subscribed to its type. +/// +/// Shared rather than cloned: a `block` or `cfilter` carries its whole payload, and the +/// pump would otherwise deep-copy it for every subscriber. See `spawn_pump`'s fan-out. +type Inbound = (SocketAddr, Arc); +type Subscribers = Arc>>>>; + +/// The kinds of peer message a sync manager can subscribe to. Replaces the +/// stringly-typed command names: managers declare interest with these variants +/// and the pump routes incoming messages by mapping `cmd()` back to one. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum MessageType { + Headers, + Inv, + CfHeaders, + CFilter, + Block, + MnListDiff, + QrInfo, + Tx, + IsDLock, + ChainLock, +} + +impl MessageType { + /// The wire command string this type corresponds to. + pub fn cmd(self) -> &'static str { + match self { + MessageType::Headers => "headers", + MessageType::Inv => "inv", + MessageType::CfHeaders => "cfheaders", + MessageType::CFilter => "cfilter", + MessageType::Block => "block", + MessageType::MnListDiff => "mnlistdiff", + MessageType::QrInfo => "qrinfo", + MessageType::Tx => "tx", + MessageType::IsDLock => "isdlock", + MessageType::ChainLock => "clsig", + } + } + + /// Map an incoming message's command back to a subscribed type, if any. + pub fn from_cmd(cmd: &str) -> Option { + Some(match cmd { + "headers" => MessageType::Headers, + "inv" => MessageType::Inv, + "cfheaders" => MessageType::CfHeaders, + "cfilter" => MessageType::CFilter, + "block" => MessageType::Block, + "mnlistdiff" => MessageType::MnListDiff, + "qrinfo" => MessageType::QrInfo, + "tx" => MessageType::Tx, + "isdlock" => MessageType::IsDLock, + "clsig" => MessageType::ChainLock, + _ => return None, + }) + } +} + +#[derive(Clone, Debug)] +pub enum NetworkEvent { + PeersUpdated, + PeerConnected(SocketAddr), + PeerDisconnected(SocketAddr), + /// A pipeline request just left the router for a peer (it is now on the + /// wire). Pipelines use this to start the request's response timeout from the + /// actual send, not from when it was queued (which fires spuriously when the + /// router is backed up). + RequestOnFlight(RequestKey), +} + +/// Identifies a pipeline request that has been sent, so the owning pipeline can +/// match it to its in-flight entry and start the timeout. One variant per +/// router-paced request type. +#[derive(Clone, Debug)] +pub enum RequestKey { + /// `getheaders` — keyed by the locator's first hash (the segment tip). + Headers(dashcore::BlockHash), + /// `getcfheaders` — keyed by the stop hash. + CfHeaders(dashcore::BlockHash), + /// `getcfilters` — keyed by the start height. + CFilters(u32), + /// `getmnlistdiff` — keyed by the target block hash. + MnListDiff(dashcore::BlockHash), + /// Block `getdata` — keyed by the requested block hash. + Block(dashcore::BlockHash), +} + +pub struct PeerNetworkManager { + connected_peers: Arc>>, + other_peers: Arc>>, + banned_peers: Vec, + discoverer: PeerDiscoverer, + msg_queue: Arc, + router: JoinHandle<()>, + pump: JoinHandle<()>, + inbound_tx: UnboundedSender, + subscribers: Subscribers, + events_tx: broadcast::Sender, + best_tip: u32, + max_peers: usize, + // Cancelled by `stop()` to tear down the router, pump and every peer reader. + shutdown: CancellationToken, +} + +/// What kind of work a queued message is, for scheduling. A single FIFO let one +/// pipeline monopolise the router: a filters phase queues batches by the hundred, +/// so a `getcfheaders` (or a small control message) landing behind them waited for +/// the whole backlog — the phases then advance one after another instead of +/// together. The router drains `Other` first and round-robins the three bulk +/// classes, so every pipeline keeps making progress. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum MsgClass { + /// Small/control messages (mempool, tx, ping, chainlock/islock `getdata`…): + /// always first, they are few and something is usually blocked on them. + Other, + Headers, + CfHeaders, + CFilters, + /// Block `getdata` — the wallet's matched blocks. + Blocks, +} + +fn classify(msg: &NetworkMessage) -> MsgClass { + match msg { + NetworkMessage::GetHeaders(_) | NetworkMessage::GetHeaders2(_) => MsgClass::Headers, + NetworkMessage::GetCFHeaders(_) => MsgClass::CfHeaders, + NetworkMessage::GetCFilters(_) => MsgClass::CFilters, + NetworkMessage::GetData(inv) + if !inv.is_empty() && inv.iter().all(|i| matches!(i, Inventory::Block(_))) => + { + MsgClass::Blocks + } + _ => MsgClass::Other, + } +} + +/// One queue per class, each behind its own lock: a pipeline enqueuing a burst +/// only contends with itself, never with the others. +struct MsgQueue { + other: Mutex>, + headers: Mutex>, + cfheaders: Mutex>, + cfilters: Mutex>, + blocks: Mutex>, + + /// Round-robin cursor over the three bulk classes. + turn: AtomicUsize, + len: AtomicUsize, + notify: Notify, +} + +struct State {} + +impl PeerNetworkManager { + pub async fn new(config: &ClientConfig) -> Self { + let mut discoverer = PeerDiscoverer::new(config); + let max_peers = config.max_peers.max(1) as usize; + + let connected_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); + let other_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); + let msg_queue = Arc::new(MsgQueue::new()); + + let (inbound_tx, inbound_rx) = mpsc::unbounded_channel(); + let subscribers: Subscribers = Arc::new(Mutex::new(HashMap::new())); + // Sized generously: `RequestOnFlight` is emitted per sent request, so the + // network-event bus is higher-volume than the peer-state events alone. + let (events_tx, _) = broadcast::channel(4096); + let shutdown = CancellationToken::new(); + // Total bytes read from all peers (download-only) and the global in-flight + // budget. The budget is NOT a fixed number: it starts at a tiny bootstrap + // just large enough to begin measuring, then `spawn_bandwidth_controller` + // sizes it from the measured download capacity (Little's Law). + let bytes = Arc::new(AtomicU64::new(0)); + let global_cap = Arc::new(AtomicUsize::new(max_peers.saturating_mul(4).max(8))); + + let best_tip = probe_and_select( + &mut discoverer, + &connected_peers, + &other_peers, + max_peers, + &inbound_tx, + &shutdown, + &bytes, + ) + .await; + + let pump = spawn_pump( + inbound_rx, + subscribers.clone(), + connected_peers.clone(), + events_tx.clone(), + msg_queue.clone(), + shutdown.clone(), + ); + + let router = spawn_router( + msg_queue.clone(), + connected_peers.clone(), + other_peers.clone(), + inbound_tx.clone(), + shutdown.clone(), + bytes.clone(), + global_cap.clone(), + events_tx.clone(), + ); + + spawn_bandwidth_controller( + bytes.clone(), + global_cap.clone(), + connected_peers.clone(), + shutdown.clone(), + ); + + let _ = events_tx.send(NetworkEvent::PeersUpdated); + + PeerNetworkManager { + connected_peers, + other_peers, + banned_peers: Vec::with_capacity(30), + discoverer, + msg_queue, + router, + pump, + inbound_tx, + subscribers, + events_tx, + best_tip, + max_peers, + shutdown, + } + } + + /// Tear down the network layer: stop the router and pump, and cancel every + /// peer reader so no more messages arrive. Called on client shutdown. + pub fn stop(&self) { + tracing::info!(target: "dash_spv::network2", "network manager stopping: cancelling tasks and peers"); + self.shutdown.cancel(); + } + + pub async fn send(&self, msg: NetworkMessage) { + self.msg_queue.push(msg).await; + } + + /// Note that `n` streaming requests served by `peer` have fully completed + /// (e.g. a `getcfilters` batch whose last `cfilter` just arrived), freeing + /// that peer's in-flight units and waking the router. Single-response + /// requests are freed in the peer's own reader instead. + pub async fn request_completed(&self, peer: SocketAddr, n: usize) { + if n == 0 { + return; + } + if let Some((p, _)) = + self.connected_peers.lock().await.iter().find(|(p, _)| p.addr() == peer) + { + p.response_completed(n); + } + self.msg_queue.notify.notify_one(); + } + + pub fn broadcast(&self, msg: NetworkMessage) { + let peers = self.connected_peers.clone(); + tokio::spawn(async move { + let guard = peers.lock().await; + for (peer, _) in guard.iter() { + let _ = peer.send(&msg).await; + } + }); + } + + /// Inject a message into the local pump as if it arrived from a peer, so + /// managers process it through the same path. Uses the `0.0.0.0:0` sentinel + /// address that managers treat as locally-originated. + pub async fn dispatch_local(&self, msg: NetworkMessage) { + let local: SocketAddr = ([0, 0, 0, 0], 0).into(); + let _ = self.inbound_tx.send(PeerEvent::Message(local, msg)); + } + + pub async fn subscribe(&self, kinds: &[MessageType]) -> UnboundedReceiver { + let (tx, rx) = mpsc::unbounded_channel(); + let mut subscribers = self.subscribers.lock().await; + for kind in kinds { + subscribers.entry(*kind).or_default().push(tx.clone()); + } + rx + } + + pub fn tip(&self) -> u32 { + self.best_tip + } + + pub fn events(&self) -> broadcast::Receiver { + self.events_tx.subscribe() + } +} + +fn spawn_router( + queue: Arc, + connected: Arc>>, + others: Arc>>, + inbound: UnboundedSender, + shutdown: CancellationToken, + bytes: Arc, + global_cap: Arc, + events: broadcast::Sender, +) -> JoinHandle<()> { + tokio::spawn(async move { + loop { + if shutdown.is_cancelled() { + break; + } + // Wait for work. `notify` fires both when a message is queued and + // when a response frees a peer slot. + if queue.len() == 0 { + tokio::select! { + _ = shutdown.cancelled() => break, + _ = queue.notify.notified() => continue, + } + } + + // Auto-connect more peers while the send queue is backed up: a deep + // queue means the connected peers can't serve our requests as fast as + // the pipelines produce them, so recruiting more raises the aggregate + // serving capacity and drains it — pushing the bottleneck toward OUR + // download link. Growth is bounded by `MAX_CONNECTED_PEERS`. Safe to + // grow past the initial `max_peers` now that each peer self-throttles + // to its MEASURED cap (a slow newcomer gets a tiny cap instead of + // feeding the old over-commit spiral). + if queue.len() > HIGH_DEMAND_QUEUE && connected.lock().await.len() < MAX_CONNECTED_PEERS + { + add_peers( + &connected, + &others, + &inbound, + &shutdown, + &bytes, + ADD_PEERS_BATCH, + MAX_CONNECTED_PEERS, + ) + .await; + } + + let peers = connected.lock().await; + let sent = + route_tick(&queue, &peers, global_cap.load(Ordering::Relaxed), &events).await; + drop(peers); + + if sent == 0 { + // Queue non-empty but every peer is at its in-flight cap: wait for + // a response to free capacity (the pump notifies on each response) + // — but never wait forever, because some of those "in-flight" + // requests may be dead. A peer that stops answering keeps its + // in-flight slots raised (the pipeline times the request out and + // re-queues it, but the peer is never told), so with enough dead + // requests every peer looks full, no response is coming, and the + // router would sleep on a notify that never fires — a hard sync + // stall with a backed-up queue. On the timeout we reclaim those + // slots and drop the peers that leaked them: they have had far + // longer than the pipelines' own timeout to answer, and leaving + // them connected would just route the freed work straight back to + // the peer that ignored it (the router picks the emptiest peer). + tokio::select! { + _ = shutdown.cancelled() => break, + _ = queue.notify.notified() => {}, + _ = tokio::time::sleep(STALL_CHECK) => { + let mut peers = connected.lock().await; + let before = peers.len(); + let mut reclaimed = 0; + peers.retain(|(p, _)| { + let n = p.reap_stale(STALE_REQUEST); + if n == 0 { + return true; + } + reclaimed += n; + // Drop the peer only if it has NEVER answered anything — + // that one is dead weight, and leaving it connected would + // route the freed work straight back to it (the router + // picks the emptiest peer). A peer that HAS served us is + // merely slow: a 2MB block over a weak link legitimately + // takes a while, and dropping those cost us 14 of 16 + // peers mid-sync. We only wanted the slot back. + let (completed, _) = p.latency_totals(); + if completed == 0 { + p.close(); + false + } else { + true + } + }); + if reclaimed > 0 { + tracing::warn!( + target: "dash_spv::network2", + "reclaimed {} stalled request slot(s); dropped {} never-responding peer(s) -> {} connected", + reclaimed, + before - peers.len(), + peers.len(), + ); + } + } + } + } + } + }) +} + +/// Extract the pipeline key of a router-paced request, so the router can report +/// it as on-flight. Returns `None` for non-pipeline messages. +fn request_keys(msg: &NetworkMessage) -> Vec { + match msg { + NetworkMessage::GetHeaders(m) | NetworkMessage::GetHeaders2(m) => { + m.locator_hashes.first().map(|h| RequestKey::Headers(*h)).into_iter().collect() + } + NetworkMessage::GetCFHeaders(m) => vec![RequestKey::CfHeaders(m.stop_hash)], + NetworkMessage::GetCFilters(m) => vec![RequestKey::CFilters(m.start_height)], + NetworkMessage::GetMnListD(m) => vec![RequestKey::MnListDiff(m.block_hash)], + // One `getdata` may name several blocks; each is its own tracked request. + NetworkMessage::GetData(inv) => inv + .iter() + .filter_map(|i| match i { + Inventory::Block(h) => Some(RequestKey::Block(*h)), + _ => None, + }) + .collect(), + _ => Vec::new(), + } +} + +async fn route_tick( + queue: &MsgQueue, + peers: &[(ConnectedPeer, State)], + global_cap: usize, + events: &broadcast::Sender, +) -> usize { + if peers.is_empty() { + return 0; + } + + // Free capacity this round = min(sum of per-peer room, global room). Each + // peer's cap is its MEASURED serving capacity (its bandwidth-delay product), + // sized by the controller from that peer's own completion rate and service + // time — fast peers carry more, slow peers less, with no fixed constant. The + // global cap is our measured download capacity. Whichever binds first limits + // this round, so we ride each peer's real ceiling without over-committing. + let total_in_flight: usize = peers.iter().map(|(p, _)| p.in_flight()).sum(); + let per_peer_room: usize = + peers.iter().map(|(p, _)| p.cap().saturating_sub(p.in_flight())).sum(); + let global_room = global_cap.saturating_sub(total_in_flight); + let capacity = per_peer_room.min(global_room); + if capacity == 0 { + return 0; + } + + let msgs = queue.pop_n(capacity).await; + let mut sent = 0; + // Anything popped that we could not put on the wire goes BACK on the queue. + // Dropping it would strand the owning pipeline forever: it has already marked + // the request as handed to the network, and its response timeout only starts + // when the router reports the request on the wire, so a dropped message is + // never re-sent and never times out. + let mut unsent: Vec = Vec::new(); + let mut msgs = msgs.into_iter(); + for msg in msgs.by_ref() { + // Send to the peer with the most free measured capacity. + let Some((peer, _)) = peers + .iter() + .filter(|(p, _)| p.in_flight() < p.cap()) + .max_by_key(|(p, _)| p.cap().saturating_sub(p.in_flight())) + else { + unsent.push(msg); // every peer is at its measured cap + break; + }; + if peer.send(&msg).await.is_ok() { + sent += 1; + // Tell the owning pipeline this request is now on the wire so it can + // start the response timeout from here, not from when it was queued. + for key in request_keys(&msg) { + let _ = events.send(NetworkEvent::RequestOnFlight(key)); + } + } else { + tracing::warn!(target: "dash_spv::network2", "router: send to {} failed", peer.addr()); + unsent.push(msg); + } + } + unsent.extend(msgs); // whatever the loop never reached + queue.push_front_all(unsent).await; + + if sent > 0 { + tracing::debug!( + target: "dash_spv::network2", + "router: sent {} | peers={} queue={}", + sent, + peers.len(), + queue.len(), + ); + } + sent +} + +/// Sizes the global in-flight budget from MEASURED download capacity — no fixed +/// magic number, per the design goal of estimating how many requests the current +/// network can absorb before it saturates. +/// +/// The estimate is Little's Law applied to downloads: the number of requests in +/// flight that sustains a completion rate `λ` at an uncongested per-request +/// service time `W` is `L = λ · W`. We measure both from the peers' response +/// stream — `λ` = requests completed per second, `W` = average time from send to +/// the response that completes the request (for `getcfilters`, dominated by the +/// download time of its ~1000 `cfilter`s, i.e. our downlink). We track the +/// minimum `W` as the uncongested baseline (the pipe's true latency at the front +/// of the knee) and target `L = λ · min_W`, probing slightly past it. +/// +/// Saturation is detected by `W` INFLATION, not by throughput: once in-flight +/// exceeds the bandwidth-delay product, extra requests just queue at the peers, +/// so `W` climbs while `λ` (and the download rate) plateaus. That inflation is +/// visible even though a raw throughput meter can't see the knee (a prior +/// throughput hill-climb ran away and regressed sync ~8x because the backlog +/// kept bytes flowing). When `W > min_W · INFLATE` we stop probing and shrink +/// back toward the sustaining level, so we ride just below saturation. +fn spawn_bandwidth_controller( + bytes: Arc, + cap: Arc, + connected: Arc>>, + shutdown: CancellationToken, +) -> JoinHandle<()> { + const WINDOW: Duration = Duration::from_millis(500); + const FLOOR_PER_PEER: usize = 2; // global floor = peers · this + const PEER_CEIL: usize = 32; // per-connection sanity bound on in-flight + const RISE: f64 = 1.05; // throughput must climb 5% to justify a bigger cap + const DROP: f64 = 0.85; // throughput below this·last => over-commit, back off + const REPROBE: u32 = 8; // plateau windows to hold before nudging the cap up + const IDLE_BPS: f64 = 1.0e6; // downlink under 1 MB/s = idle, hold the cap + const EMA_ALPHA: f64 = 0.5; // smoothing for the noisy per-window rate + let window_s = WINDOW.as_secs_f64(); + + tokio::spawn(async move { + let mut last_bytes = bytes.load(Ordering::Relaxed); + let mut rate_ema = 0.0f64; // smoothed downlink bytes/s + let mut last_rate = 0.0f64; // smoothed rate at the previous cap adjustment + let mut hold = 0u32; // consecutive plateau windows + let mut ticker = tokio::time::interval(WINDOW); + loop { + tokio::select! { + _ = shutdown.cancelled() => break, + _ = ticker.tick() => {} + } + + // Downlink throughput (bytes read off the sockets — our downlink only). + // This is the saturation signal: unlike per-request service time — which + // balloons with cfilter payload size and out-of-order batch completion, + // reading "saturated" forever and pinning the cap at its floor — bytes/s + // directly reflects whether we are using the pipe. We grow the in-flight + // budget while throughput keeps climbing with it and stop at the plateau + // (the bandwidth-delay product: past it, more in-flight only grows queues, + // not bytes/s), backing off if it collapses (peers over-committed). + let now_bytes = bytes.load(Ordering::Relaxed); + let dl_rate = now_bytes.saturating_sub(last_bytes) as f64 / window_s; + last_bytes = now_bytes; + rate_ema = if rate_ema == 0.0 { + dl_rate + } else { + EMA_ALPHA * dl_rate + (1.0 - EMA_ALPHA) * rate_ema + }; + + let npeers = connected.lock().await.len(); + if npeers == 0 { + continue; + } + let floor = (npeers * FLOOR_PER_PEER).max(FLOOR_PER_PEER); + let ceiling = (npeers * PEER_CEIL).max(floor + 1); + let step = npeers.max(4); // ~one extra slot per peer per window + let cur = cap.load(Ordering::Relaxed); + + // Gradient hill-climb on smoothed throughput. + let (new, action) = if rate_ema < IDLE_BPS { + // Nothing meaningful downloading (e.g. the commit tail): hold the + // budget steady so it is ready when the download resumes. + (cur, "idle") + } else if cur <= floor || rate_ema >= last_rate * RISE { + // Still gaining (or at the floor): push the budget up. + hold = 0; + ((cur + step).min(ceiling), "grow") + } else if rate_ema < last_rate * DROP { + // Throughput collapsed — the peers are over-committed. Back off. + hold = 0; + (((cur as f64 * 0.8) as usize).max(floor), "backoff") + } else { + // Plateau: we are at the knee. Hold, re-probing up occasionally to + // catch a capacity increase (a faster peer, less congestion). + hold += 1; + if hold >= REPROBE { + hold = 0; + ((cur + step).min(ceiling), "reprobe") + } else { + (cur, "hold") + } + }; + // Anchor RISE/DROP to the rate at each real adjustment (skip idle/hold + // windows) so the next comparison is like-for-like. + if matches!(action, "grow" | "backoff" | "reprobe") { + last_rate = rate_ema; + } + cap.store(new, Ordering::Relaxed); + + // Spread the global budget evenly across peers with a slot of headroom, + // so the GLOBAL cap is the binding constraint (route_tick fills the + // emptiest peer first, so fast peers naturally serve more) while no + // single connection is over-committed beyond PEER_CEIL. + let per_peer = (new.div_ceil(npeers) + 1).clamp(FLOOR_PER_PEER, PEER_CEIL); + { + let g = connected.lock().await; + for (p, _) in g.iter() { + p.set_cap(per_peer); + } + } + + tracing::debug!( + target: "dash_spv::network2", + "bandwidth: {:.1} MB/s (ema {:.1}) | {} | global cap={} | peers={} per-peer cap={}", + dl_rate / 1e6, + rate_ema / 1e6, + action, + new, + npeers, + per_peer, + ); + } + }) +} + +async fn add_peers( + connected: &Mutex>, + others: &Mutex>, + inbound: &UnboundedSender, + shutdown: &CancellationToken, + bytes: &Arc, + batch: usize, + max_peers: usize, +) { + // Never grow past the operating cap; only backfill the deficit toward it. + let deficit = max_peers.saturating_sub(connected.lock().await.len()); + if deficit == 0 { + return; + } + let candidates: Vec = { + let mut others = others.lock().await; + let take = batch.min(deficit).min(others.len()); + others.drain(..take).collect() + }; + + let mut added = 0; + for candidate in candidates { + if let Ok(peer) = candidate.connect(inbound.clone(), shutdown.clone(), bytes.clone()).await + { + connected.lock().await.push((peer, State {})); + added += 1; + } + } + + if added > 0 { + let total = connected.lock().await.len(); + tracing::info!( + target: "dash_spv::network2", + "high demand: added {} peers -> {} connected", + added, + total, + ); + } +} + +async fn probe_and_select( + discoverer: &mut PeerDiscoverer, + connected: &Mutex>, + others: &Mutex>, + max_peers: usize, + inbound: &UnboundedSender, + shutdown: &CancellationToken, + bytes: &Arc, +) -> u32 { + const CONNECT_CHUNK: usize = 16; // bounded concurrent handshakes per round + const FAST_LAG_MS: u32 = 100; // prefer peers that ping under this + + // Sort key for latency: lower is better; treat an unmeasured lag (0) as worst. + fn lag_key(p: &ConnectedPeer) -> u32 { + match p.lag_ms() { + 0 => u32::MAX, + ms => ms, + } + } + + let mut candidates = discoverer.get(PROBE_BATCH).await; + // Cap how many candidates we handshake before settling. This bounds startup: + // real peers often ping over 100ms, so without a limit the "wait for fast + // peers" loop would probe the entire batch. We try a few times `max_peers` + // and then take the lowest-latency of whatever connected. + let attempt_max = (max_peers * 3).max(CONNECT_CHUNK * 2); + let mut attempted = 0usize; + let mut kept: Vec = Vec::new(); + let mut fallback: Vec = Vec::new(); // connected, not fast enough + let mut tip = 0u32; + + // Connect a chunk at a time and STOP as soon as we have `max_peers` fast + // peers (or we've tried enough), so sync starts without waiting on the whole + // probe batch. We keep only the peers we'll actually query connected; + // everything else stays a bare address for on-demand recruitment. + while kept.len() < max_peers && !candidates.is_empty() && attempted < attempt_max { + let take = CONNECT_CHUNK.min(candidates.len()); + attempted += take; + let batch: Vec = candidates.drain(..take).collect(); + let results = join_all( + batch.into_iter().map(|c| c.connect(inbound.clone(), shutdown.clone(), bytes.clone())), + ) + .await; + for peer in results.into_iter().filter_map(Result::ok) { + tip = tip.max(peer.version().start_height.max(0) as u32); + let lag = peer.lag_ms(); + if kept.len() < max_peers && lag > 0 && lag < FAST_LAG_MS { + kept.push(peer); + } else { + fallback.push(peer); + } + } + } + + // If too few peers were fast, fill up to `max_peers` with the lowest-latency + // fallbacks; close and remember the rest as backups. + fallback.sort_by_key(lag_key); + while kept.len() < max_peers && !fallback.is_empty() { + kept.push(fallback.remove(0)); + } + let mut backups: Vec = candidates; // un-probed addresses + for peer in fallback { + peer.close(); // drop the connection we won't use + backups.push(peer.disconnect()); + } + + let fast = kept.iter().filter(|p| p.lag_ms() > 0 && p.lag_ms() < FAST_LAG_MS).count(); + tracing::info!( + target: "dash_spv::network2", + "probe: connected {} peers ({} fast <{}ms), {} backups | tip={}", + kept.len(), + fast, + FAST_LAG_MS, + backups.len(), + tip, + ); + + *connected.lock().await = kept.into_iter().map(|peer| (peer, State {})).collect(); + *others.lock().await = backups; + + tip +} + +fn spawn_pump( + mut inbound: UnboundedReceiver, + subscribers: Subscribers, + connected: Arc>>, + events: broadcast::Sender, + queue: Arc, + shutdown: CancellationToken, +) -> JoinHandle<()> { + tokio::spawn(async move { + // Per-peer received-message counter to check load balance across peers. + let mut recv_by_peer: HashMap = HashMap::new(); + let mut total_recv: u64 = 0; + loop { + let t_idle = std::time::Instant::now(); + let event = tokio::select! { + _ = shutdown.cancelled() => break, + ev = inbound.recv() => match ev { + Some(ev) => ev, + None => break, + }, + }; + crate::timer::P_PUMP_IDLE.add(t_idle.elapsed()); + match event { + PeerEvent::Message(addr, msg) => { + let t_busy = std::time::Instant::now(); + *recv_by_peer.entry(addr).or_insert(0) += 1; + total_recv += 1; + if total_recv % 250_000 == 0 { + let mut dist: Vec<(SocketAddr, u64)> = + recv_by_peer.iter().map(|(a, c)| (*a, *c)).collect(); + dist.sort_by_key(|(_, c)| std::cmp::Reverse(*c)); + tracing::info!( + target: "filt_depth", + "[t+{}ms] recv balance ({} peers, {} total): {:?}", + crate::timer::elapsed_ms(), + dist.len(), + total_recv, + dist, + ); + // Per-peer request->response latency (avg / worst). + let lat: Vec<(SocketAddr, u64, String)> = connected + .lock() + .await + .iter() + .map(|(p, _)| { + let (n, avg, max) = p.latency_stats(); + (p.addr(), n, format!("avg={avg:.1}ms max={max:.1}ms")) + }) + .collect(); + tracing::info!( + target: "filt_depth", + "[t+{}ms] peer latency: {:?}", + crate::timer::elapsed_ms(), + lat, + ); + } + let mt = MessageType::from_cmd(msg.cmd()); + // Single-message responses free a slot in the peer's reader; + // wake the router so it can use the freed capacity. `cfilter` + // is skipped — its batch frees a slot via `request_completed`, + // which notifies once per ~1000 messages instead of each. + if mt != Some(MessageType::CFilter) { + queue.notify.notify_one(); + } + + { + let mut subscribers = subscribers.lock().await; + if let Some(list) = mt.and_then(|t| subscribers.get_mut(&t)) { + let last = list.len().saturating_sub(1); + let mut msg = Some(Arc::new(msg)); + let mut idx = 0; + + list.retain(|tx| { + // Hand the *last* subscriber our own reference instead of a + // clone. Every heavy message type (block, cfilter, cfheaders, + // headers) has exactly one subscriber, so it arrives with a + // refcount of 1 and the manager can take the payload without + // copying it. Only `inv`, which is tiny, is really shared. + let shared = if idx == last { + msg.take().expect("taken once, on the final subscriber") + } else { + Arc::clone( + msg.as_ref().expect("held until the final subscriber"), + ) + }; + idx += 1; + + tx.send((addr, shared)).is_ok() + }); + } + } + crate::timer::P_PUMP_BUSY.add(t_busy.elapsed()); + } + PeerEvent::Disconnected(addr) => { + let remaining = { + let mut guard = connected.lock().await; + guard.retain(|(peer, _)| peer.addr() != addr); + guard.len() + }; + tracing::info!( + target: "dash_spv::network2", + "peer disconnected: {} | {} peers remaining", + addr, + remaining, + ); + let _ = events.send(NetworkEvent::PeerDisconnected(addr)); + } + } + } + }) +} + +impl MsgQueue { + fn new() -> Self { + Self { + other: Mutex::new(VecDeque::with_capacity(30)), + headers: Mutex::new(VecDeque::with_capacity(30)), + cfheaders: Mutex::new(VecDeque::with_capacity(30)), + cfilters: Mutex::new(VecDeque::with_capacity(30)), + blocks: Mutex::new(VecDeque::with_capacity(30)), + turn: AtomicUsize::new(0), + len: AtomicUsize::new(0), + notify: Notify::new(), + } + } + + fn len(&self) -> usize { + self.len.load(Ordering::SeqCst) + } + + fn queue(&self, class: MsgClass) -> &Mutex> { + match class { + MsgClass::Other => &self.other, + MsgClass::Headers => &self.headers, + MsgClass::CfHeaders => &self.cfheaders, + MsgClass::CFilters => &self.cfilters, + MsgClass::Blocks => &self.blocks, + } + } + + /// Take up to `n` messages in scheduling order: all pending control traffic + /// first (it is small, and something is usually blocked on it), then the bulk + /// classes by WEIGHTED rotation. + /// + /// Filters get most of the turns because they are the phase that actually costs + /// time: a compact filter is ~700 bytes per block against 32 for a filter + /// header, so the filter download is what the sync waits on. + /// + /// But not ALL the turns — strict priority would be a trap. Filters can only be + /// verified (and therefore stored, scanned and committed) once the filter-header + /// chain has been verified through their range, so starving cfheaders behind a + /// deep cfilters backlog stalls the very phase we were trying to favour. The + /// weights keep the small, cheap requests flowing while filters take the rest. + async fn pop_n(&self, n: usize) -> Vec { + if n == 0 { + return Vec::new(); + } + let mut out: Vec = Vec::with_capacity(n); + + { + let mut other = self.other.lock().await; + let take = n.min(other.len()); + out.extend(other.drain(..take)); + } + + // One lap of the rotation: filters four turns for every one of each header + // class. A class with an empty queue simply yields its turn. + // Filters take half the lap (they are the bytes), blocks a quarter (they + // gate the gap-limit cascade, so stalling them stretches the tail), and the + // two small header classes an eighth each — enough to keep the chains that + // unblock the filters moving. + const ROTATION: [MsgClass; 8] = [ + MsgClass::CFilters, + MsgClass::Blocks, + MsgClass::CFilters, + MsgClass::Headers, + MsgClass::CFilters, + MsgClass::Blocks, + MsgClass::CFilters, + MsgClass::CfHeaders, + ]; + let mut turn = self.turn.load(Ordering::Relaxed); + while out.len() < n { + let mut got = false; + for _ in 0..ROTATION.len() { + if out.len() == n { + break; + } + let class = ROTATION[turn % ROTATION.len()]; + turn = turn.wrapping_add(1); + if let Some(msg) = self.queue(class).lock().await.pop_front() { + out.push(msg); + got = true; + } + } + if !got { + break; // every bulk class is empty + } + } + self.turn.store(turn, Ordering::Relaxed); + + self.len.fetch_sub(out.len(), Ordering::SeqCst); + out + } + + async fn push(&self, msg: NetworkMessage) { + self.queue(classify(&msg)).lock().await.push_back(msg); + self.len.fetch_add(1, Ordering::SeqCst); + self.notify.notify_one(); + } + + /// Return messages that were popped but could not be sent to the FRONT of + /// their own class queue, preserving order. A popped message must never be + /// dropped: the owning pipeline has already recorded it as handed to the + /// network and only starts its response timeout once the router reports it on + /// the wire, so a dropped message is one the pipeline waits on forever — a + /// permanent sync stall (observed as: queue backed up, 0 MB/s, no sends, no + /// timeouts). + async fn push_front_all(&self, msgs: Vec) { + if msgs.is_empty() { + return; + } + let n = msgs.len(); + for msg in msgs.into_iter().rev() { + self.queue(classify(&msg)).lock().await.push_front(msg); + } + self.len.fetch_add(n, Ordering::SeqCst); + self.notify.notify_one(); + } +} diff --git a/dash-spv/src/network2/mod.rs b/dash-spv/src/network2/mod.rs new file mode 100644 index 000000000..15bedd101 --- /dev/null +++ b/dash-spv/src/network2/mod.rs @@ -0,0 +1,5 @@ +mod discovery; +mod manager; +mod peer; + +pub use manager::{MessageType, NetworkEvent, PeerNetworkManager, RequestKey}; diff --git a/dash-spv/src/network2/peer.rs b/dash-spv/src/network2/peer.rs new file mode 100644 index 000000000..6cb66f524 --- /dev/null +++ b/dash-spv/src/network2/peer.rs @@ -0,0 +1,536 @@ +use std::collections::VecDeque; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex as StdMutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +/// Per-peer response-latency tracker: the time each pipeline request spends in +/// flight, from send to the response that completes it. Send times are queued +/// FIFO; each completing response pops the oldest and folds the elapsed time +/// into running total/count/max (so we can report avg and worst per peer). +#[derive(Default)] +pub(crate) struct Latency { + pending: StdMutex>, + total_ns: AtomicU64, + count: AtomicU64, + max_ns: AtomicU64, +} + +impl Latency { + fn on_send(&self) { + self.pending.lock().unwrap().push_back(Instant::now()); + } + + /// Drop pending sends older than `timeout` (the pipelines have long since + /// timed them out and re-queued them elsewhere), returning how many. Their + /// in-flight slots would otherwise be held forever — see + /// [`ConnectedPeer::reap_stale`]. + fn reap(&self, timeout: Duration) -> usize { + let mut pending = self.pending.lock().unwrap(); + let now = Instant::now(); + let mut n = 0; + while let Some(&front) = pending.front() { + if now.duration_since(front) > timeout { + pending.pop_front(); + n += 1; + } else { + break; // FIFO: the rest are younger + } + } + n + } + + /// Pop the oldest pending send and record its round-trip. + fn complete_one(&self) { + let sent = self.pending.lock().unwrap().pop_front(); + if let Some(sent) = sent { + let ns = sent.elapsed().as_nanos() as u64; + self.total_ns.fetch_add(ns, Ordering::Relaxed); + self.count.fetch_add(1, Ordering::Relaxed); + self.max_ns.fetch_max(ns, Ordering::Relaxed); + } + } + + /// Cumulative (completed request count, total service-time nanoseconds). + /// Deltas between two reads give the windowed completion rate and average + /// service time used to size the in-flight budget by Little's Law. + fn totals(&self) -> (u64, u64) { + (self.count.load(Ordering::Relaxed), self.total_ns.load(Ordering::Relaxed)) + } + + /// (completed request count, average ms, worst ms). + fn snapshot(&self) -> (u64, f64, f64) { + let count = self.count.load(Ordering::Relaxed); + let total = self.total_ns.load(Ordering::Relaxed); + let max = self.max_ns.load(Ordering::Relaxed); + let avg_ms = if count > 0 { + total as f64 / count as f64 / 1e6 + } else { + 0.0 + }; + (count, avg_ms, max as f64 / 1e6) + } +} + +use dashcore::{ + consensus::encode, + network::{ + address::Address, + constants::ServiceFlags, + message::{NetworkMessage, RawNetworkMessage, RawNetworkMessageCodec}, + message_network::VersionMessage, + }, + Network, +}; +use futures::lock::Mutex; +use tokio::sync::mpsc::UnboundedSender; +use tokio::{ + io::{AsyncRead, AsyncWriteExt, ReadBuf}, + net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, + }, +}; +use tokio_stream::StreamExt; +use tokio_util::codec::FramedRead; +use tokio_util::sync::CancellationToken; + +use crate::{error::NetworkResult, NetworkError}; + +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); +const USER_AGENT: &str = concat!("/dash-spv:", env!("CARGO_PKG_VERSION"), "/"); + +/// Wraps a socket read half and adds every byte read into a shared counter, so +/// the network manager can estimate download throughput (and size the global +/// in-flight budget to ~90% of it). +struct CountingReader { + inner: R, + bytes: Arc, +} + +impl AsyncRead for CountingReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> std::task::Poll> { + let before = buf.filled().len(); + let r = std::pin::Pin::new(&mut self.inner).poll_read(cx, buf); + if let std::task::Poll::Ready(Ok(())) = &r { + let n = buf.filled().len() - before; + self.bytes.fetch_add(n as u64, Ordering::Relaxed); + } + r + } +} + +type PeerReader = FramedRead, RawNetworkMessageCodec>; + +pub enum PeerEvent { + Message(SocketAddr, NetworkMessage), + Disconnected(SocketAddr), +} + +pub struct ConnectedPeer { + network: Network, + addr: SocketAddr, + version: VersionMessage, + lag_ms: AtomicU32, + in_flight: Arc, + latency: Arc, + writer: Arc>, + /// This peer's measured serving capacity — the number of requests it can have + /// in flight before ITS responses start queuing (its bandwidth-delay product). + /// Sized continuously by the bandwidth controller from the peer's own + /// completion rate and service time, mirroring how the global budget is sized + /// from our download rate. Fast peers earn a high cap, slow peers a low one. + cap: Arc, + /// Per-connection cancel token (child of the global shutdown). Cancelling it + /// stops this peer's reader and closes the socket. Used to drop peers we + /// probed but don't keep, so we only hold connections we actually use. + token: CancellationToken, +} + +pub struct DisconnectedPeer { + network: Network, + addr: SocketAddr, +} + +impl ConnectedPeer { + pub fn addr(&self) -> SocketAddr { + self.addr + } + + pub fn version(&self) -> &VersionMessage { + &self.version + } + + /// Net in-flight to this peer: `+1` per message we send it, `-1` per message + /// we read from it (managed internally by `send` and the reader task). The + /// router reads this to send to the least-loaded peer. + pub fn in_flight(&self) -> usize { + self.in_flight.load(Ordering::Relaxed) + } + + pub fn disconnect(self) -> DisconnectedPeer { + DisconnectedPeer { + network: self.network, + addr: self.addr, + } + } + + pub async fn send(&self, msg: &NetworkMessage) -> NetworkResult<()> { + // TODO: Take a reference to msg instead of cloning it + let raw = RawNetworkMessage { + magic: self.network.magic(), + payload: msg.clone(), + }; + let serialized = encode::serialize(&raw); + + if let Err(e) = self.writer.lock().await.write_all(&serialized).await { + tracing::warn!("Disconnecting {} due to write error: {}", self.addr, e); + return Err(NetworkError::ConnectionFailed(format!("Write failed: {}", e))); + } + // A pipeline request counts as one unit of in-flight work for this peer. + if is_pipeline_request(msg) { + self.in_flight.fetch_add(1, Ordering::Relaxed); + self.latency.on_send(); + } + Ok(()) + } + + /// Note that `n` earlier pipeline requests have fully completed, freeing that + /// much in-flight work. Used for streaming responses (`getcfilters` -> many + /// `cfilter`s) that the reader can't attribute to a finished request on its + /// own; single-response requests are decremented directly in the reader. + pub(crate) fn response_completed(&self, n: usize) { + let _ = self + .in_flight + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(n))); + for _ in 0..n { + self.latency.complete_one(); + } + } + + /// Reclaim the in-flight slots of requests this peer has not answered within + /// `timeout`, returning how many. Without this they leak: when a pipeline + /// times a request out it re-queues it, but nothing tells the peer its + /// request died, so the peer's `in_flight` counter stays raised forever. + /// Enough dead requests and EVERY peer sits at its cap with nothing actually + /// on the wire — the router sees zero free capacity, stops sending, and the + /// sync deadlocks with a full queue (reproduced at 32 peers: 0 MB/s, queue + /// 140, no sends). The count also identifies the peer as unresponsive, so the + /// caller can drop it rather than keep handing it work. + pub(crate) fn reap_stale(&self, timeout: Duration) -> usize { + let n = self.latency.reap(timeout); + if n > 0 { + let _ = self + .in_flight + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(n))); + } + n + } + + /// Per-peer response latency: (completed requests, average ms, worst ms). + pub(crate) fn latency_stats(&self) -> (u64, f64, f64) { + self.latency.snapshot() + } + + /// Cumulative (completed request count, total service-time ns) for this peer. + /// The bandwidth controller diffs these across a window to get the download + /// completion rate and service time that size the in-flight budget. + pub(crate) fn latency_totals(&self) -> (u64, u64) { + self.latency.totals() + } + + /// Handshake round-trip latency in ms (0 if unmeasured). + pub(crate) fn lag_ms(&self) -> u32 { + self.lag_ms.load(Ordering::Relaxed) + } + + /// Close this connection: cancel its reader so the socket shuts down. Used to + /// drop probed-but-unselected peers instead of leaking their readers. + pub(crate) fn close(&self) { + self.token.cancel(); + } + + /// This peer's current measured in-flight capacity (its serving BDP). + pub(crate) fn cap(&self) -> usize { + self.cap.load(Ordering::Relaxed) + } + + /// Update this peer's measured in-flight capacity (called by the controller). + pub(crate) fn set_cap(&self, n: usize) { + self.cap.store(n, Ordering::Relaxed); + } +} + +/// Pipeline requests we send and expect a response for (each adds one in-flight). +fn is_pipeline_request(msg: &NetworkMessage) -> bool { + match msg { + NetworkMessage::GetHeaders(_) + | NetworkMessage::GetHeaders2(_) + | NetworkMessage::GetCFHeaders(_) + | NetworkMessage::GetCFilters(_) => true, + // Block `getdata` is paced like any other request: the blocks pipeline + // sends ONE block per message, so a request is exactly one in-flight unit + // and one `block` in reply. Other `getdata` (chainlocks, islocks, txs) is + // control traffic — it carries several inventory items whose replies the + // reader cannot attribute one-for-one, so counting it would leak slots. + NetworkMessage::GetData(inv) => { + !inv.is_empty() + && inv + .iter() + .all(|i| matches!(i, dashcore::network::message_blockdata::Inventory::Block(_))) + } + _ => false, + } +} + +/// Single-message responses: the reader decrements one in-flight per message. +/// `cfilter` is excluded — one `getcfilters` yields up to 1000 `cfilter`s, so its +/// unit is freed once per batch by the filters pipeline via `response_completed`. +fn is_single_response(msg: &NetworkMessage) -> bool { + matches!( + msg, + NetworkMessage::Headers(_) + | NetworkMessage::Headers2(_) + | NetworkMessage::CFHeaders(_) + | NetworkMessage::Block(_) + ) +} + +impl DisconnectedPeer { + pub fn new(addr: SocketAddr, network: Network) -> Self { + DisconnectedPeer { + network, + addr, + } + } + + pub async fn connect( + self, + inbound: UnboundedSender, + shutdown: CancellationToken, + bytes: Arc, + ) -> NetworkResult { + let stream = TcpStream::connect(&self.addr).await.map_err(|e| { + NetworkError::ConnectionFailed(format!("Failed to connect to {}: {}", self.addr, e)) + })?; + + let (read_half, mut writer) = stream.into_split(); + let mut reader = FramedRead::new( + CountingReader { + inner: read_half, + bytes, + }, + RawNetworkMessageCodec, + ); + let magic = self.network.magic(); + + handshake_send(&mut writer, magic, NetworkMessage::Version(build_version(self.addr))) + .await?; + + let deadline = tokio::time::Instant::now() + HANDSHAKE_TIMEOUT; + let mut peer_version: Option = None; + let mut got_verack = false; + + while !(peer_version.is_some() && got_verack) { + let raw = match tokio::time::timeout_at(deadline, reader.next()).await { + Err(_) => return Err(NetworkError::Timeout), + Ok(None) => return Err(NetworkError::PeerDisconnected), + Ok(Some(Err(e))) => return Err(e.into()), + Ok(Some(Ok(raw))) => raw, + }; + if raw.magic != magic { + return Err(NetworkError::ProtocolError("wrong network magic".into())); + } + match raw.payload { + NetworkMessage::Version(v) => { + // BIP155: sendaddrv2 must be sent BEFORE verack. + handshake_send(&mut writer, magic, NetworkMessage::SendAddrV2).await?; + handshake_send(&mut writer, magic, NetworkMessage::Verack).await?; + peer_version = Some(v); + } + NetworkMessage::Verack => got_verack = true, + NetworkMessage::Ping(n) => { + handshake_send(&mut writer, magic, NetworkMessage::Pong(n)).await? + } + _ => {} + } + } + + let version = peer_version.ok_or(NetworkError::PeerDisconnected)?; + + // Announce sendheaders only after the handshake is fully complete. + handshake_send(&mut writer, magic, NetworkMessage::SendHeaders).await?; + + // Measure round-trip lag with a post-handshake ping/pong. Sending a ping + // before the handshake completes makes some peers drop us, so we do it here. + let mut lag_ms: u32 = 0; + let ping_nonce: u64 = rand::random(); + let ping_sent = tokio::time::Instant::now(); + if handshake_send(&mut writer, magic, NetworkMessage::Ping(ping_nonce)).await.is_ok() { + let deadline = ping_sent + HANDSHAKE_TIMEOUT; + while let Ok(Some(Ok(raw))) = tokio::time::timeout_at(deadline, reader.next()).await { + if raw.magic != magic { + continue; + } + match raw.payload { + NetworkMessage::Pong(n) if n == ping_nonce => { + lag_ms = ping_sent.elapsed().as_millis().clamp(1, u32::MAX as u128) as u32; + break; + } + NetworkMessage::Ping(n) => { + let _ = handshake_send(&mut writer, magic, NetworkMessage::Pong(n)).await; + } + _ => {} + } + } + } + + let writer = Arc::new(Mutex::new(writer)); + let in_flight = Arc::new(AtomicUsize::new(0)); + let latency = Arc::new(Latency::default()); + // Start with a tiny in-flight capacity; the controller grows it from this + // peer's measured serving rate. + let cap = Arc::new(AtomicUsize::new(2)); + // Per-connection token: cancelled by the global shutdown (parent) OR by + // `close()` to drop just this peer. + let token = shutdown.child_token(); + spawn_reader( + self.addr, + magic, + reader, + writer.clone(), + inbound, + in_flight.clone(), + latency.clone(), + token.clone(), + ); + + tracing::debug!( + target: "dash_spv::network2", + "peer connected: {} | lag={}ms height={}", + self.addr, + lag_ms, + version.start_height, + ); + + Ok(ConnectedPeer { + network: self.network, + addr: self.addr, + version, + lag_ms: AtomicU32::new(lag_ms), + in_flight, + latency, + writer, + cap, + token, + }) + } +} + +fn spawn_reader( + addr: SocketAddr, + magic: u32, + mut reader: PeerReader, + writer: Arc>, + inbound: UnboundedSender, + in_flight: Arc, + latency: Arc, + shutdown: CancellationToken, +) { + tokio::spawn(async move { + loop { + let t_recv = std::time::Instant::now(); + let next = tokio::select! { + _ = shutdown.cancelled() => break, + next = reader.next() => next, + }; + crate::timer::P_READ_RECV.add(t_recv.elapsed()); + match next { + None => break, + Some(Err(e)) => { + tracing::error!("NETWORK: reader {} stopped: {}", addr, e); + break; + } + Some(Ok(raw)) => { + if raw.magic != magic { + continue; + } + // A single-message response completes one unit of in-flight + // work. Streaming responses (cfilter) are freed per batch by + // the filters pipeline instead. + if is_single_response(&raw.payload) { + let _ = in_flight.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| { + Some(v.saturating_sub(1)) + }); + latency.complete_one(); + } + match raw.payload { + NetworkMessage::Ping(nonce) => { + let pong = RawNetworkMessage { + magic, + payload: NetworkMessage::Pong(nonce), + }; + if writer + .lock() + .await + .write_all(&encode::serialize(&pong)) + .await + .is_err() + { + break; + } + } + payload => { + if inbound.send(PeerEvent::Message(addr, payload)).is_err() { + break; + } + } + } + } + } + } + + tracing::info!("NETWORK: peer {} disconnected", addr); + let _ = inbound.send(PeerEvent::Disconnected(addr)); + }); +} + +fn build_version(peer: SocketAddr) -> VersionMessage { + let now = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0); + let unspecified: SocketAddr = ([0u8, 0, 0, 0], 0).into(); + VersionMessage::new( + ServiceFlags::NONE, + now, + Address::new(&peer, ServiceFlags::NETWORK), + Address::new(&unspecified, ServiceFlags::NONE), + rand::random(), + USER_AGENT.to_string(), + 0, + false, + [0u8; 32], + ) +} + +async fn handshake_send( + writer: &mut OwnedWriteHalf, + magic: u32, + payload: NetworkMessage, +) -> NetworkResult<()> { + let raw = RawNetworkMessage { + magic, + payload, + }; + writer + .write_all(&encode::serialize(&raw)) + .await + .map_err(|e| NetworkError::ConnectionFailed(format!("handshake write failed: {}", e)))?; + writer + .flush() + .await + .map_err(|e| NetworkError::ConnectionFailed(format!("handshake flush failed: {}", e)))?; + Ok(()) +} diff --git a/dash-spv/src/storage/segments.rs b/dash-spv/src/storage/segments.rs index 49f291add..e54286c93 100644 --- a/dash-spv/src/storage/segments.rs +++ b/dash-spv/src/storage/segments.rs @@ -28,20 +28,50 @@ pub trait Persistable: Sized + Encodable + Decodable + PartialEq + Clone { const SEGMENT_PREFIX: &'static str = "segment"; const DATA_FILE_EXTENSION: &'static str = "dat"; + /// Soft ceiling on the memory a [`SegmentCache`] of this type may hold. + /// + /// A segment is a fixed array of [`Segment::ITEMS_PER_SEGMENT`] slots, so its + /// footprint is `size_of::()` per slot plus whatever the stored items own + /// on the heap. Capping the number of resident *segments* therefore says nothing + /// about the bytes they hold: 10 segments of `FilterHeader` are 16 MB, while 10 + /// dense segments of block filters are hundreds of MB. Each type sets its own + /// budget here and the cache evicts least-recently-used segments to stay under it. + /// + /// This is a soft ceiling: [`SegmentCache::MIN_RESIDENT_SEGMENTS`] segments always + /// stay resident, so a single oversized segment can still overshoot it. + const CACHE_BUDGET_BYTES: usize = 32 * 1024 * 1024; + fn segment_file_name(segment_id: u32) -> String { format!("{}_{:04}.{}", Self::SEGMENT_PREFIX, segment_id, Self::DATA_FILE_EXTENSION) } fn sentinel() -> Self; + + /// Bytes this item owns on the heap, on top of the `size_of::()` its slot + /// already costs. Sentinels must report 0: an empty segment owns no heap. + fn heap_bytes(&self) -> usize { + 0 + } } impl Persistable for Vec { + /// Block filters. The only cache whose payload dwarfs its slots: a dense segment + /// holds 50k real filters, so the budget — not the segment count — is what bounds it. + const CACHE_BUDGET_BYTES: usize = 48 * 1024 * 1024; + fn sentinel() -> Self { vec![] } + + fn heap_bytes(&self) -> usize { + self.capacity() + } } impl Persistable for HashedBlockHeader { + /// 112 B per slot, so ~5.6 MB per segment of pure slots and no heap payload. + const CACHE_BUDGET_BYTES: usize = 24 * 1024 * 1024; + fn sentinel() -> Self { let header = BlockHeader { version: Version::from_consensus(i32::MAX), // Invalid version @@ -57,12 +87,19 @@ impl Persistable for HashedBlockHeader { } impl Persistable for FilterHeader { + /// 32 B per slot: the cheapest cache, ~1.6 MB per segment and no heap payload. + const CACHE_BUDGET_BYTES: usize = 8 * 1024 * 1024; + fn sentinel() -> Self { FilterHeader::from_byte_array([0u8; 32]) } } impl Persistable for HashedBlock { + /// Sparse: only filter-matched blocks land here, but every segment still + /// materializes 50k slots (~6.8 MB) whether or not it holds a single block. + const CACHE_BUDGET_BYTES: usize = 48 * 1024 * 1024; + fn sentinel() -> Self { let block = Block { header: *HashedBlockHeader::sentinel().header(), @@ -70,6 +107,20 @@ impl Persistable for HashedBlock { }; Self::from(block) } + + fn heap_bytes(&self) -> usize { + let block = self.block(); + + // The sentinel is an empty block, and an empty `txdata` owns no allocation. + if block.txdata.is_empty() { + return 0; + } + + // Serialized size, used as a proxy for the transactions' true heap footprint. + // It undercounts the per-`Vec` overhead inside each transaction, so treat the + // resulting budget as approximate rather than exact. + block.size() + } } /// In-memory cache for all segments of items @@ -86,13 +137,19 @@ pub struct SegmentCache { } impl SegmentCache { - const MAX_ACTIVE_SEGMENTS: usize = 10; + /// Segments kept resident regardless of [`Persistable::CACHE_BUDGET_BYTES`]. + /// + /// Must stay at 2 or more. During a rescan each cache has two hot regions — the + /// segment the download is appending at the tip, and the segment the scan is + /// reading behind it — and evicting down to one would make those two regions + /// evict each other, reloading a 50k-item segment from disk on every alternation. + const MIN_RESIDENT_SEGMENTS: usize = 2; pub async fn load_or_new(segments_dir: impl Into) -> StorageResult { let segments_dir = segments_dir.into(); let mut cache = Self { - segments: HashMap::with_capacity(Self::MAX_ACTIVE_SEGMENTS), + segments: HashMap::new(), evicted: HashMap::new(), tip_height: None, start_height: None, @@ -162,31 +219,69 @@ impl SegmentCache { Ok(&*segment) } + /// Memory currently held by this cache: every resident segment, plus any segment + /// parked in `evicted` because its eviction write failed. + /// + /// `O(resident segments)`, and each segment reports its size from a running + /// counter rather than walking its items. + pub fn resident_bytes(&self) -> usize { + let resident: usize = self.segments.values().map(Segment::resident_bytes).sum(); + let parked: usize = self.evicted.values().map(Segment::resident_bytes).sum(); + + resident + parked + } + + /// Evict least-recently-used segments until the cache fits in its byte budget. + /// + /// `protect` is a segment the caller is about to hand out a reference to; it is + /// never chosen for eviction. Without it, a segment resurrected from `evicted` + /// carries its old `last_accessed` and could be picked as the LRU on the very call + /// that resurrected it. + /// + /// A dirty segment is written to disk on its way out, rather than being parked in + /// `evicted` until the next `persist`. `evicted` is unbounded, so parking there is + /// reserved for segments whose write failed — otherwise a fast sync could stack up + /// arbitrarily many dirty segments between the worker's 5-second persists. + async fn enforce_budget(&mut self, protect: Option) { + while self.segments.len() > Self::MIN_RESIDENT_SEGMENTS + && self.resident_bytes() > I::CACHE_BUDGET_BYTES + { + let lru = self + .segments + .iter() + .filter(|(k, _)| Some(**k) != protect) + .min_by_key(|(_, s)| s.last_accessed) + .map(|(k, _)| *k); + + let Some(key) = lru else { + break; + }; + let Some(mut segment) = self.segments.remove(&key) else { + break; + }; + + if segment.state == SegmentState::Dirty { + if let Err(e) = segment.persist(&self.segments_dir).await { + // Dropping it would lose the writes. Park it and let the next + // `persist` retry; it still counts against `resident_bytes`, but + // the `segments.len()` guard above keeps this loop terminating. + tracing::error!("Failed to persist evicted segment {key}: {e}"); + self.evicted.insert(key, segment); + } + } + } + } + async fn get_segment_mut<'a>( &'a mut self, segment_id: &u32, ) -> StorageResult<&'a mut Segment> { - let segments_len = self.segments.len(); - if self.segments.contains_key(segment_id) { let segment = self.segments.get_mut(segment_id).expect("We already checked that it exists"); return Ok(segment); } - if segments_len >= Self::MAX_ACTIVE_SEGMENTS { - let key_to_evict = - self.segments.iter_mut().min_by_key(|(_, s)| s.last_accessed).map(|(k, v)| (*k, v)); - - if let Some((key, _)) = key_to_evict { - if let Some(segment) = self.segments.remove(&key) { - if segment.state == SegmentState::Dirty { - self.evicted.insert(key, segment); - } - } - } - } - // If the segment is already in the to_persist map, load it from there. // If the segment is queued for deletion, return a fresh empty segment. // The next `persist` will atomically overwrite the stale file. @@ -199,7 +294,17 @@ impl SegmentCache { Segment::load(&self.segments_dir, *segment_id).await? }; - let segment = self.segments.entry(*segment_id).or_insert(segment); + self.segments.insert(*segment_id, segment); + + // Trim *after* inserting, so the cache is back under budget by the time this + // call returns. Trimming first would leave it one segment over the ceiling + // until the next miss happened to trim again. + self.enforce_budget(Some(*segment_id)).await; + + let segment = self + .segments + .get_mut(segment_id) + .expect("the segment was just inserted and is protected from eviction"); Ok(segment) } @@ -369,6 +474,11 @@ impl SegmentCache { None => Some(start_height), }; + // Storing into a segment already resident grows it without ever missing, so the + // eviction in `get_segment_mut` never runs. A dense segment of block filters + // gains tens of MB this way; re-check the budget once the batch has landed. + self.enforce_budget(None).await; + Ok(()) } @@ -494,6 +604,10 @@ pub struct Segment { items: Vec, state: SegmentState, last_accessed: Instant, + /// Running sum of `I::heap_bytes()` across `items`, kept up to date by every + /// mutation so [`Self::resident_bytes`] stays O(1) instead of walking 50k slots + /// on each budget check. + heap_bytes: usize, } impl Segment { @@ -503,14 +617,23 @@ impl Segment { debug_assert!(items.len() <= Self::ITEMS_PER_SEGMENT as usize); items.resize(Self::ITEMS_PER_SEGMENT as usize, I::sentinel()); + let heap_bytes = items.iter().map(I::heap_bytes).sum(); + Self { segment_id, items, state, last_accessed: Instant::now(), + heap_bytes, } } + /// Total memory this segment holds: its fixed slot array plus the heap owned by + /// the items in it. + fn resident_bytes(&self) -> usize { + self.items.capacity() * std::mem::size_of::() + self.heap_bytes + } + pub fn first_valid_offset(&self) -> Option { let sentinel = I::sentinel(); @@ -606,14 +729,18 @@ impl Segment { let sentinel = I::sentinel(); let start = (offset as usize) + 1; let mut changed = false; + let mut freed = 0usize; for slot in &mut self.items[start..] { if *slot != sentinel { + freed += slot.heap_bytes(); *slot = sentinel.clone(); changed = true; } } + self.heap_bytes = self.heap_bytes.saturating_sub(freed); + if changed { self.state = SegmentState::Dirty; self.last_accessed = Instant::now(); @@ -631,6 +758,11 @@ impl Segment { // we are not storing an already valid and stored item. debug_assert!(self.items[offset] == I::sentinel()); + // Account for the outgoing slot too: the debug_assert above says it is a + // sentinel (which owns no heap), but that only holds in debug builds. + self.heap_bytes = self.heap_bytes.saturating_sub(self.items[offset].heap_bytes()); + self.heap_bytes += item.heap_bytes(); + self.items[offset] = item; self.state = SegmentState::Dirty; @@ -676,38 +808,124 @@ mod tests { use super::*; + /// Segments are evicted by bytes, not by count, and a dirty segment is written to + /// disk on its way out rather than parked in `evicted` until the next `persist`. #[tokio::test] - async fn test_segment_cache_eviction() { + async fn test_segment_cache_evicts_to_stay_under_budget() { let tmp_dir = TempDir::new().unwrap(); - const MAX_SEGMENTS: u32 = SegmentCache::::MAX_ACTIVE_SEGMENTS as u32; + // A FilterHeader segment is 50k * 32 B = 1.6 MB, so this many segments + // overshoot the type's budget several times over and force evictions. + const SEGMENTS: u32 = 16; let mut cache = SegmentCache::::load_or_new(tmp_dir.path()) .await .expect("Failed to create new segment_cache"); - // This logic is a little tricky. Each cache can contain up to MAX_SEGMENTS segments in memory. - // By storing MAX_SEGMENTS + 1 items, we ensure that the cache will evict the first introduced. - // Then, by asking again in order starting in 0, we force the cache to load the evicted segment - // evicting at the same time the next, 1 in this case. Then we ask for the 1 that we know is - // evicted and so on. - - for i in 0..=MAX_SEGMENTS { + for i in 0..SEGMENTS { let segment = cache.get_segment_mut(&i).await.expect("Failed to create a new segment"); assert!(segment.state == SegmentState::Dirty); segment.insert(FilterHeader::dummy(i), 0); } - for i in 0..=MAX_SEGMENTS { - assert_eq!(cache.segments.len(), MAX_SEGMENTS as usize); + assert!( + cache.segments.len() < SEGMENTS as usize, + "nothing was evicted: all {SEGMENTS} segments are still resident" + ); + + // Dirty segments left on the way out, so nothing is stranded in memory waiting + // for the 5-second persist worker. + assert!(cache.evicted.is_empty(), "an eviction parked a segment instead of writing it"); + // Reading every segment back walks the cache past its budget again, so this + // also exercises loading evicted segments from disk while evicting others. + for i in 0..SEGMENTS { let segment = cache.get_segment_mut(&i).await.expect("Failed to load segment"); assert_eq!(segment.get(0..1), [FilterHeader::dummy(i)]); } } + /// The budget is a real ceiling while the cache is being *written to*, not just + /// after a miss: `insert` grows a resident segment without ever missing, which is + /// exactly how a dense run of block filters gets big. + #[tokio::test] + async fn test_resident_bytes_stays_under_budget_while_storing() { + let tmp_dir = TempDir::new().unwrap(); + + const ITEMS_PER_SEGMENT: u32 = Segment::>::ITEMS_PER_SEGMENT; + const BUDGET: usize = as Persistable>::CACHE_BUDGET_BYTES; + + // 256 B per filter, in the range a real mainnet basic filter occupies. A dense + // segment is then ~14 MB, so several fit in the budget and eviction — not the + // MIN_RESIDENT_SEGMENTS floor — is what bounds the cache here. + let filters: Vec> = + (0..ITEMS_PER_SEGMENT * 4).map(|i| vec![i as u8; 256]).collect(); + + let mut cache = + SegmentCache::>::load_or_new(tmp_dir.path()).await.expect("Failed to create"); + + for (chunk, items) in filters.chunks(10_000).enumerate() { + let start = chunk as u32 * 10_000; + cache.store_items_at_height(items, start).await.expect("Failed to store items"); + + assert!( + cache.resident_bytes() <= BUDGET, + "resident {} B exceeds budget {BUDGET} B after storing at height {start}", + cache.resident_bytes(), + ); + } + + // The count-based cap this replaced would have held all four segments resident + // (10 was the limit) and never looked at what they weighed. + assert!(cache.segments.len() < 4, "no segment was evicted"); + + // Eviction wrote the filters out on the way, so they survive the round trip. + cache.persist(tmp_dir.path()).await; + + let mut reloaded = + SegmentCache::>::load_or_new(tmp_dir.path()).await.expect("Failed to reload"); + + assert_eq!(reloaded.get_items(0..10).await.unwrap(), filters[0..10]); + + let last = (ITEMS_PER_SEGMENT * 3) as usize; + assert_eq!( + reloaded.get_items(last as u32..last as u32 + 10).await.unwrap(), + filters[last..last + 10] + ); + } + + /// A single segment whose payload alone exceeds the budget cannot be evicted below + /// the MIN_RESIDENT_SEGMENTS floor. The cache overshoots — the budget is a soft + /// ceiling — but it must still settle at the floor rather than growing unbounded. + #[tokio::test] + async fn test_oversized_segments_settle_at_the_resident_floor() { + let tmp_dir = TempDir::new().unwrap(); + + const ITEMS_PER_SEGMENT: u32 = Segment::>::ITEMS_PER_SEGMENT; + + // 1 KB per filter: one dense segment is ~52 MB, over the budget on its own. + let filters: Vec> = + (0..ITEMS_PER_SEGMENT * 4).map(|i| vec![i as u8; 1024]).collect(); + + let mut cache = + SegmentCache::>::load_or_new(tmp_dir.path()).await.expect("Failed to create"); + + for (chunk, items) in filters.chunks(10_000).enumerate() { + cache + .store_items_at_height(items, chunk as u32 * 10_000) + .await + .expect("Failed to store items"); + } + + assert_eq!( + cache.segments.len(), + SegmentCache::>::MIN_RESIDENT_SEGMENTS, + "an oversized segment must still evict down to the floor" + ); + } + #[tokio::test] async fn test_segment_cache_persist_load() { let tmp_dir = TempDir::new().unwrap(); diff --git a/dash-spv/src/sync/block_headers/manager.rs b/dash-spv/src/sync/block_headers/manager.rs index b3699e979..9844a93cb 100644 --- a/dash-spv/src/sync/block_headers/manager.rs +++ b/dash-spv/src/sync/block_headers/manager.rs @@ -13,7 +13,7 @@ use std::time::Instant; use crate::chain::CheckpointManager; use crate::error::{SyncError, SyncResult}; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, BlockHeaderTip, MetadataStorage}; use crate::sync::block_headers::HeadersPipeline; use crate::sync::{BlockHeadersProgress, ProgressPercentage, SyncEvent, SyncManager, SyncState}; @@ -123,7 +123,7 @@ impl BlockHeadersManager { pub(super) async fn handle_headers_pipeline( &mut self, headers: &[Header], - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { if !self.pipeline.is_initialized() { // Pipeline not initialized (shouldn't happen in normal flow) @@ -134,8 +134,35 @@ impl BlockHeadersManager { let was_syncing = self.state() == SyncState::Syncing; let tip_was_complete = self.pipeline.is_tip_complete(); - // Route headers to the pipeline, validates checkpoint match. - let matched = self.pipeline.receive_headers(headers)?; + // Hash the batch off this task: block_hash() is X11, the dominant block-header + // cost (~70ms per 8000-header batch), and it otherwise runs serially on this + // single manager task. Split into 4 chunks across tokio's runtime workers, then + // feed the pre-hashed headers to the pipeline (validates checkpoint match). + let matched = if headers.is_empty() { + self.pipeline.receive_headers_prehashed(&[])? + } else { + let len = headers.len(); + let chunk = len.div_ceil(4).max(1); + let mut set = tokio::task::JoinSet::new(); + let (mut idx, mut start) = (0usize, 0usize); + while start < len { + let end = (start + chunk).min(len); + let c: Vec
= headers[start..end].to_vec(); + let i = idx; + set.spawn( + async move { (i, c.iter().map(HashedBlockHeader::from).collect::>()) }, + ); + start = end; + idx += 1; + } + let mut parts: Vec<(usize, Vec)> = Vec::with_capacity(idx); + while let Some(res) = set.join_next().await { + parts.push(res.map_err(|e| SyncError::InvalidState(format!("hash join: {e}")))?); + } + parts.sort_by_key(|(i, _)| *i); + let hashed: Vec = parts.into_iter().flat_map(|(_, v)| v).collect(); + self.pipeline.receive_headers_prehashed(&hashed)? + }; if matched.is_none() && !headers.is_empty() { tracing::debug!( @@ -147,14 +174,25 @@ impl BlockHeadersManager { // Send more requests during initial sync or active post-sync catch-up. // Skip for unsolicited headers. if was_syncing || !tip_was_complete { - let sent = self.pipeline.send_pending(requests)?; + let sent = self.pipeline.send_pending(network).await?; if sent > 0 { tracing::debug!("Pipeline sent {} more requests", sent); } } - // Process ready-to-store segments + // Announce the batch that just landed in memory (per segment, before the + // in-order store) so filter-header sync can request its cfheaders in + // parallel without waiting for the sequential store head. let mut events = Vec::new(); + if let Some((_, start_height, end_height, stop_hash)) = matched { + events.push(SyncEvent::BlockHeadersInMemory { + start_height, + end_height, + stop_hash, + }); + } + + // Process ready-to-store segments let ready_batches = self.pipeline.take_ready_to_store(); for (_start_height, batch_headers) in ready_batches { @@ -199,7 +237,7 @@ impl BlockHeadersManager { self.pending_announcements.len() ); self.pipeline.reset_tip_segment(); - self.pipeline.send_pending(requests)?; + self.pipeline.send_pending(network).await?; } else { // Synced to the tip and no pending announcements, finalize and emit event let tip = self.tip().await?; @@ -229,7 +267,7 @@ impl BlockHeadersManager { pub(super) async fn handle_inventory( &mut self, inv: &[Inventory], - _requests: &RequestSender, + _network: &Arc, ) -> SyncResult<()> { for inv_item in inv { if let Inventory::Block(block_hash) = inv_item { @@ -253,312 +291,3 @@ impl BlockHeadersManager { Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::chain::checkpoints::testnet_checkpoints; - use crate::network::{MessageType, NetworkEvent, NetworkRequest, RequestSender}; - use crate::storage::{ - DiskStorageManager, PersistentBlockHeaderStorage, PersistentMetadataStorage, StorageManager, - }; - use crate::sync::{ManagerIdentifier, SyncManager, SyncManagerProgress}; - use dashcore::network::message::NetworkMessage; - use tokio::sync::mpsc::unbounded_channel; - - type TestBlockHeadersManager = - BlockHeadersManager; - - fn create_test_checkpoint_manager() -> Arc { - Arc::new(CheckpointManager::new(testnet_checkpoints())) - } - - async fn create_test_manager() -> TestBlockHeadersManager { - let mut storage = DiskStorageManager::with_temp_dir().await.unwrap(); - // Store a genesis header so the manager can initialize - let genesis = Header::dummy_batch(0..1); - storage - .store_headers( - &genesis.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - let checkpoint_manager = create_test_checkpoint_manager(); - BlockHeadersManager::new(storage.block_headers(), storage.metadata(), checkpoint_manager) - .await - .expect("Failed to create BlockHeadersManager") - } - - /// Create a manager in synced state with an initialized pipeline. - async fn create_synced_manager() -> TestBlockHeadersManager { - let mut manager = create_test_manager().await; - let tip = manager.tip().await.unwrap(); - manager.pipeline.init(tip.height(), *tip.hash(), tip.height()); - manager.progress.set_state(SyncState::Synced); - manager - } - - #[tokio::test] - async fn test_block_headers_manager_new() { - let manager = create_test_manager().await; - assert_eq!(manager.identifier(), ManagerIdentifier::BlockHeader); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - assert_eq!(manager.wanted_message_types(), vec![MessageType::Headers, MessageType::Inv]); - } - - #[tokio::test] - async fn test_headers_manager_progress() { - let mut manager = create_test_manager().await; - manager.progress.update_tip_height(100); - manager.progress.update_target_height(200); - manager.progress.add_processed(50); - - let progress = manager.progress(); - if let SyncManagerProgress::BlockHeaders(progress) = progress { - assert_eq!(progress.state(), SyncState::WaitingForConnections); - assert_eq!(progress.tip_height(), 100); - assert_eq!(progress.target_height(), 200); - assert_eq!(progress.processed(), 50); - assert!(progress.last_activity().elapsed().as_secs() < 1); - } else { - panic!("Expected SyncManagerProgress::BlockHeaders"); - } - } - - #[tokio::test] - async fn test_headers_manager_has_pipeline() { - let manager = create_test_manager().await; - assert!(!manager.pipeline.is_initialized()); - assert_eq!(manager.pipeline.segment_count(), 0); - } - - fn create_test_request_sender( - ) -> (RequestSender, tokio::sync::mpsc::UnboundedReceiver) { - let (tx, rx) = unbounded_channel(); - (RequestSender::new(tx), rx) - } - - #[tokio::test] - async fn test_unsolicited_post_sync_header_does_not_trigger_get_headers() { - let mut manager = create_test_manager().await; - let tip = manager.tip().await.unwrap(); - let tip_hash = *tip.hash(); - - // Simulate completed sync: pipeline initialized with tip segment marked complete - manager.pipeline.init(0, tip_hash, 0); - manager.pipeline.mark_tip_complete(); - manager.progress.set_state(SyncState::Synced); - - let (sender, mut rx) = create_test_request_sender(); - - let header = Header::dummy_chain(1, tip_hash).remove(0); - - let events = manager.handle_headers_pipeline(&[header], &sender).await.unwrap(); - - // Header should have been stored - assert_eq!(events.len(), 1); - assert!(matches!( - events[0], - SyncEvent::BlockHeadersStored { - tip_height: 1 - } - )); - - // No GetHeaders request should have been sent - assert!(rx.try_recv().is_err()); - - // Tip segment marked complete again for the next unsolicited header - assert!(manager.pipeline.is_tip_complete()); - } - - #[tokio::test] - async fn test_peer_tip_announcement_lifecycle() { - let mut manager = create_synced_manager().await; - let (requests, mut rx) = create_test_request_sender(); - - let addr: SocketAddr = "1.2.3.4:9999".parse().unwrap(); - let connect = NetworkEvent::PeerConnected { - address: addr, - }; - - // Connect sends a peer-targeted GetHeaders - let events = manager.handle_network_event(&connect, &requests).await.unwrap(); - assert!(events.is_empty()); - assert!(manager.announced_peers.contains(&addr)); - match rx.try_recv().unwrap() { - NetworkRequest::SendMessageToPeer(_, target_addr) => { - assert_eq!(target_addr, addr); - } - other => panic!("Expected SendMessageToPeer, got {:?}", other), - } - - // Same peer again sends nothing (already announced) - manager.handle_network_event(&connect, &requests).await.unwrap(); - assert!(rx.try_recv().is_err()); - - // Disconnect removes from announced set - let disconnect = NetworkEvent::PeerDisconnected { - address: addr, - }; - manager.handle_network_event(&disconnect, &requests).await.unwrap(); - assert!(!manager.announced_peers.contains(&addr)); - - // Reconnect sends GetHeaders again - manager.handle_network_event(&connect, &requests).await.unwrap(); - assert!(manager.announced_peers.contains(&addr)); - assert!(rx.try_recv().is_ok()); - } - - #[tokio::test] - async fn test_peer_tip_announcement_guards() { - // Not synced: peer connect does nothing - let mut manager = create_test_manager().await; - let (requests, mut rx) = create_test_request_sender(); - let addr: SocketAddr = "1.2.3.4:9999".parse().unwrap(); - let connect = NetworkEvent::PeerConnected { - address: addr, - }; - - manager.handle_network_event(&connect, &requests).await.unwrap(); - assert!(!manager.announced_peers.contains(&addr)); - assert!(rx.try_recv().is_err()); - - // Active catch-up: peer connect skipped while pipeline has pending request - let mut manager = create_synced_manager().await; - manager.pipeline.reset_tip_segment(); - manager.pipeline.send_pending(&requests).unwrap(); - rx.try_recv().unwrap(); // drain the pipeline GetHeaders - - manager.handle_network_event(&connect, &requests).await.unwrap(); - assert!(!manager.announced_peers.contains(&addr)); - assert!(rx.try_recv().is_err()); - } - - #[tokio::test] - async fn test_disconnect_preserves_pipeline_and_resumes_from_advanced_tip() { - let mut manager = create_test_manager().await; - let (requests, mut rx) = create_test_request_sender(); - - // Use a target below the first testnet checkpoint (50000) so the - // pipeline produces a single open-ended tip segment. - let initial_event = NetworkEvent::PeersUpdated { - connected_count: 1, - best_height: Some(40_000), - addresses: vec![], - }; - manager.handle_network_event(&initial_event, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Syncing); - assert!(manager.pipeline.is_initialized()); - assert_eq!(manager.pipeline.segment_count(), 1); - - let initial_locator = match rx.try_recv().expect("initial GetHeaders not sent") { - NetworkRequest::SendMessage(NetworkMessage::GetHeaders(msg)) => msg.locator_hashes[0], - other => panic!("Expected GetHeaders, got {:?}", other), - }; - assert!(rx.try_recv().is_err()); - - // Simulate a peer response. The single tip segment drains its buffer - // through take_ready_to_store, advancing the storage tip and the - // segment's current_tip_hash to advanced_hash. - let header = Header::dummy_chain(1, initial_locator).remove(0); - let advanced_hash = header.block_hash(); - manager.handle_headers_pipeline(&[header], &requests).await.unwrap(); - - // Drain the follow-up GetHeaders that send_pending issued. - match rx.try_recv().expect("follow-up GetHeaders not sent") { - NetworkRequest::SendMessage(NetworkMessage::GetHeaders(msg)) => { - assert_eq!(msg.locator_hashes[0], advanced_hash); - } - other => panic!("Expected GetHeaders, got {:?}", other), - } - assert!(rx.try_recv().is_err()); - - let disconnect_event = NetworkEvent::PeersUpdated { - connected_count: 0, - best_height: Some(40_000), - addresses: vec![], - }; - manager.handle_network_event(&disconnect_event, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - assert!( - manager.pipeline.is_initialized(), - "pipeline must survive disconnect so resume can reuse validated state" - ); - assert_eq!(manager.pipeline.segment_count(), 1); - - // Reconnect: start_sync must skip pipeline.init and resume by sending - // GetHeaders from each segment's preserved current_tip_hash. - manager.handle_network_event(&initial_event, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Syncing); - - let resumed_locator = match rx.try_recv().expect("resumed GetHeaders not sent") { - NetworkRequest::SendMessage(NetworkMessage::GetHeaders(msg)) => msg.locator_hashes[0], - other => panic!("Expected GetHeaders, got {:?}", other), - }; - assert_eq!( - resumed_locator, advanced_hash, - "GetHeaders on reconnect must use the preserved current_tip_hash" - ); - assert_ne!(resumed_locator, initial_locator); - assert!(rx.try_recv().is_err()); - } - - #[tokio::test] - async fn test_disconnect_after_sync_resumes_and_catches_up() { - let mut manager = create_synced_manager().await; - let tip = manager.tip().await.unwrap(); - let synced_hash = *tip.hash(); - manager.pipeline.mark_tip_complete(); - assert!(manager.pipeline.is_tip_complete()); - - let (requests, mut rx) = create_test_request_sender(); - - let disconnect_event = NetworkEvent::PeersUpdated { - connected_count: 0, - best_height: Some(tip.height()), - addresses: vec![], - }; - manager.handle_network_event(&disconnect_event, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - assert!(manager.pipeline.is_initialized()); - - // Reconnect with a higher peer best_height (a new block was mined). - let reconnect_event = NetworkEvent::PeersUpdated { - connected_count: 1, - best_height: Some(tip.height() + 1), - addresses: vec![], - }; - manager.handle_network_event(&reconnect_event, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Syncing); - - let resumed_locator = match rx.try_recv().expect("resumed GetHeaders not sent") { - NetworkRequest::SendMessage(NetworkMessage::GetHeaders(msg)) => msg.locator_hashes[0], - other => panic!("Expected GetHeaders, got {:?}", other), - }; - assert_eq!(resumed_locator, synced_hash); - assert!(rx.try_recv().is_err()); - } - - #[tokio::test] - async fn test_empty_headers_after_tip_announcement_is_harmless() { - let mut manager = create_synced_manager().await; - manager.pipeline.mark_tip_complete(); - let (requests, mut rx) = create_test_request_sender(); - - // Announce tip to a new peer - let addr: SocketAddr = "1.2.3.4:9999".parse().unwrap(); - let connect = NetworkEvent::PeerConnected { - address: addr, - }; - manager.handle_network_event(&connect, &requests).await.unwrap(); - rx.try_recv().unwrap(); // drain the GetHeaders request - - // Peer responds with empty headers (same height as us) - let events = manager.handle_headers_pipeline(&[], &requests).await.unwrap(); - - // No events emitted, no requests sent, tip segment stays complete - assert!(events.is_empty()); - assert!(rx.try_recv().is_err()); - assert!(manager.pipeline.is_tip_complete()); - } -} diff --git a/dash-spv/src/sync/block_headers/pipeline.rs b/dash-spv/src/sync/block_headers/pipeline.rs index cce5baca9..4cf034ac3 100644 --- a/dash-spv/src/sync/block_headers/pipeline.rs +++ b/dash-spv/src/sync/block_headers/pipeline.rs @@ -6,12 +6,13 @@ use std::sync::Arc; +#[cfg(test)] use dashcore::block::Header; use dashcore::BlockHash; use crate::chain::CheckpointManager; use crate::error::SyncResult; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::sync::block_headers::segment_state::SegmentState; use crate::types::HashedBlockHeader; @@ -117,9 +118,25 @@ impl HeadersPipeline { self.segments.len() } + /// Clear timed-out in-flight requests across all segments so `send_pending` + /// re-issues them (recovers from a lost request / dead peer). + pub fn handle_timeouts(&mut self, timeout: std::time::Duration) { + for segment in &mut self.segments { + segment.handle_timeouts(timeout); + } + } + + /// Start the response timeout for the segment whose GetHeaders (locator = + /// `hash`) the router just put on the wire. No-op on the other segments. + pub fn mark_on_flight(&mut self, hash: &BlockHash) { + for segment in &mut self.segments { + segment.coordinator.mark_on_flight(&[*hash]); + } + } + /// Send pending requests for active segments. /// Returns the number of requests sent. - pub fn send_pending(&mut self, requests: &RequestSender) -> SyncResult { + pub async fn send_pending(&mut self, network: &Arc) -> SyncResult { let mut sent = 0; for segment in &mut self.segments { // Skip completed segments @@ -127,7 +144,7 @@ impl HeadersPipeline { continue; } while segment.can_send() { - segment.send_request(requests)?; + segment.send_request(network).await?; sent += 1; } } @@ -137,8 +154,20 @@ impl HeadersPipeline { /// Try to match incoming headers to the correct segment. /// Returns the segment index if matched, or None if headers don't belong to any segment. /// Returns an error if checkpoint validation fails. + #[cfg(test)] pub fn receive_headers(&mut self, headers: &[Header]) -> SyncResult> { - if headers.is_empty() { + let hashed: Vec = headers.iter().map(HashedBlockHeader::from).collect(); + Ok(self.receive_headers_prehashed(&hashed)?.map(|(sid, ..)| sid)) + } + + /// Match an already-hashed batch to the correct segment and route it. The X11 + /// block hashing is done in parallel at the manager (on the tokio runtime); + /// this is the production path, `receive_headers` is a test-only wrapper. + pub fn receive_headers_prehashed( + &mut self, + hashed: &[HashedBlockHeader], + ) -> SyncResult> { + if hashed.is_empty() { // Empty response means the peer has no more headers after our locator. // Route to the tip segment (target_height is None) if it has in-flight requests. // Middle segments complete via checkpoint validation, not empty responses. @@ -152,14 +181,15 @@ impl HeadersPipeline { segment.segment_id, segment.current_height ); - segment.receive_headers(headers)?; - return Ok(Some(segment.segment_id)); + segment.receive_headers_prehashed(hashed)?; + // Empty tip response has no batch to announce. + return Ok(None); } } return Ok(None); } - let prev_hash = headers[0].prev_blockhash; + let prev_hash = hashed[0].header().prev_blockhash; // Find the segment that matches for (idx, segment) in self.segments.iter_mut().enumerate() { @@ -182,14 +212,15 @@ impl HeadersPipeline { segment.segment_id ); } - segment.receive_headers(headers)?; - return Ok(Some(segment.segment_id)); + let sid = segment.segment_id; + let range = segment.receive_headers_prehashed(hashed)?; + return Ok(range.map(|(start, end, stop_hash)| (sid, start, end, stop_hash))); } } // Check if these are duplicate headers from another peer. The first // header's hash matches a segment's current tip, meaning we already have it. - let first_hash = headers[0].block_hash(); + let first_hash = *hashed[0].hash(); if self.segments.iter().any(|s| s.current_tip_hash == first_hash) { tracing::debug!("Ignoring duplicate header {} from another peer", first_hash); return Ok(None); @@ -197,7 +228,7 @@ impl HeadersPipeline { tracing::warn!( "Received {} headers with prev_hash {} but no segment matched", - headers.len(), + hashed.len(), prev_hash ); Ok(None) @@ -261,13 +292,6 @@ impl HeadersPipeline { self.segments.iter().map(|s| s.buffered_headers.len() as u32).sum() } - /// Check for timeouts in all segments. - pub fn handle_timeouts(&mut self) { - for segment in &mut self.segments { - segment.handle_timeouts(); - } - } - /// Drop only per-peer in-flight bookkeeping across every segment. /// /// Buffered headers, segment topology, and per-segment validated tip state @@ -334,296 +358,3 @@ impl HeadersPipeline { .is_some_and(|s| !s.complete && s.coordinator.active_count() > 0) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::chain::checkpoints::{mainnet_checkpoints, testnet_checkpoints}; - use tokio::sync::mpsc::unbounded_channel; - - use crate::network::{NetworkRequest, RequestSender}; - use crate::sync::block_headers::segment_state::SegmentState; - - fn create_test_checkpoint_manager(is_testnet: bool) -> Arc { - let checkpoints = if is_testnet { - testnet_checkpoints() - } else { - mainnet_checkpoints() - }; - Arc::new(CheckpointManager::new(checkpoints)) - } - - fn create_test_request_sender( - ) -> (RequestSender, tokio::sync::mpsc::UnboundedReceiver) { - let (tx, rx) = unbounded_channel(); - (RequestSender::new(tx), rx) - } - - #[test] - fn test_pipeline_new() { - let cm = create_test_checkpoint_manager(true); - let pipeline = HeadersPipeline::new(cm); - - assert!(!pipeline.is_initialized()); - assert_eq!(pipeline.segment_count(), 0); - } - - #[test] - fn test_pipeline_init_testnet() { - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm.clone()); - - // Get genesis hash for testnet - let genesis = cm.get_checkpoint(0).unwrap(); - pipeline.init(0, genesis.block_hash, 1_200_000); - - assert!(pipeline.is_initialized()); - // Should have segments: 0->500k, 500k->800k, 800k->1.1M, 1.1M->tip - assert!(pipeline.segment_count() >= 3); - } - - #[test] - fn test_pipeline_init_from_middle() { - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm.clone()); - - // Start from checkpoint 500k - let cp_500k = cm.get_checkpoint(500_000).unwrap(); - pipeline.init(500_000, cp_500k.block_hash, 1_200_000); - - // Should have fewer segments since we're starting from 500k - assert!(pipeline.is_initialized()); - // Segments: 500k->800k, 800k->1.1M, 1.1M->tip - assert!(pipeline.segment_count() >= 2); - } - - #[test] - fn test_pipeline_send_pending() { - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm.clone()); - - let genesis = cm.get_checkpoint(0).unwrap(); - pipeline.init(0, genesis.block_hash, 1_200_000); - - let (sender, mut rx) = create_test_request_sender(); - - let sent = pipeline.send_pending(&sender).unwrap(); - - // Should send at least one request per segment - assert!(sent >= pipeline.segment_count()); - - // Verify messages were queued - let mut count = 0; - while rx.try_recv().is_ok() { - count += 1; - } - assert_eq!(count, sent); - } - - #[test] - fn test_pipeline_is_complete_initially() { - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm.clone()); - - let genesis = cm.get_checkpoint(0).unwrap(); - pipeline.init(0, genesis.block_hash, 1_200_000); - - assert!(!pipeline.is_complete()); - } - - #[test] - fn test_take_ready_to_store_empty() { - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm.clone()); - - let genesis = cm.get_checkpoint(0).unwrap(); - pipeline.init(0, genesis.block_hash, 1_200_000); - - let ready = pipeline.take_ready_to_store(); - assert!(ready.is_empty()); - } - - #[test] - fn test_completed_tip_segment_accepts_unsolicited_post_sync_headers() { - // After initial sync completes, peers may push new block headers without - // us requesting them. The completed tip segment should accept these - // unsolicited headers by marking them as in-flight before processing. - let tip_hash = BlockHash::dummy(99); - - let mut tip_seg = SegmentState::new(0, 1000, tip_hash, None, None); - tip_seg.complete = true; - tip_seg.current_height = 1000; - tip_seg.current_tip_hash = tip_hash; - - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm); - pipeline.initialized = true; - pipeline.segments = vec![tip_seg]; - - // Simulate an unsolicited header arriving from a peer (no in-flight request) - let mut header = Header::dummy(1); - header.prev_blockhash = tip_hash; - - let matched = pipeline.receive_headers(&[header]).unwrap(); - assert_eq!(matched, Some(0), "Tip segment should accept unsolicited post-sync headers"); - - assert!(!pipeline.segments[0].complete, "Tip segment should be reset to non-complete"); - assert_eq!(pipeline.segments[0].buffered_headers.len(), 1); - assert_eq!(pipeline.segments[0].current_height, 1001); - } - - #[test] - fn test_completed_segment_does_not_steal_next_segment_headers() { - // Create two segments which share the checkpoint hash boundary. - // - Segment 0: height 0 -> target 100 - // - Segment 1: height 100 -> target 200 - let shared_hash = BlockHash::dummy(42); - - let mut segment_0 = - SegmentState::new(0, 0, BlockHash::dummy(0), Some(100), Some(shared_hash)); - // Mark segment 0 as complete at the checkpoint — its current_tip_hash is the shared hash - segment_0.complete = true; - segment_0.current_height = 100; - segment_0.current_tip_hash = shared_hash; - - let segment_1 = SegmentState::new(1, 100, shared_hash, Some(200), None); - - // Build a pipeline with these two segments - let checkpoint_manager = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(checkpoint_manager); - pipeline.initialized = true; - pipeline.segments = vec![segment_0, segment_1]; - - // Create a header whose prev_blockhash is the shared hash - let mut header = Header::dummy(1); - header.prev_blockhash = shared_hash; - - // Mark segment 1 request as in-flight so receive works - pipeline.segments[1].coordinator.mark_sent(&[shared_hash]); - - // Route headers should go to segment 1, not the completed segment 0 - let matched = pipeline.receive_headers(&[header]).unwrap(); - assert_eq!(matched, Some(1), "Headers should route to segment 1, not completed segment 0"); - - // Segment 0 should still have no extra buffered headers - assert!(pipeline.segments[0].buffered_headers.is_empty()); - // Segment 1 should have the header - assert_eq!(pipeline.segments[1].buffered_headers.len(), 1); - } - - #[test] - fn test_duplicate_headers_from_another_peer_are_ignored() { - let tip_hash = BlockHash::dummy(50); - - let mut tip_seg = SegmentState::new(0, 1000, tip_hash, None, None); - tip_seg.current_height = 1001; - // Simulate that we already received a header and advanced the tip - let mut first_header = Header::dummy(1); - first_header.prev_blockhash = tip_hash; - let new_tip_hash = first_header.block_hash(); - tip_seg.current_tip_hash = new_tip_hash; - - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm); - pipeline.initialized = true; - pipeline.segments = vec![tip_seg]; - - // Another peer sends the same header (prev_blockhash is old tip, first - // header hash matches the segment's current tip) - let matched = pipeline.receive_headers(&[first_header]).unwrap(); - assert_eq!(matched, None, "Duplicate headers should be silently ignored"); - assert!(pipeline.segments[0].buffered_headers.is_empty()); - } - - #[test] - fn test_clear_in_flight_preserves_buffers_across_segments() { - let shared_hash = BlockHash::dummy(42); - - let mut completed = - SegmentState::new(0, 0, BlockHash::dummy(0), Some(100), Some(shared_hash)); - completed.complete = true; - completed.current_height = 100; - completed.current_tip_hash = shared_hash; - // Buffered headers on a complete-but-not-yet-drained segment must survive. - let mut completed_header = Header::dummy(1); - completed_header.prev_blockhash = BlockHash::dummy(0); - completed.buffered_headers.push(HashedBlockHeader::from(completed_header)); - - let mut mid = SegmentState::new(1, 100, shared_hash, Some(200), None); - mid.coordinator.mark_sent(&[shared_hash]); - let mut mid_header = Header::dummy(2); - mid_header.prev_blockhash = shared_hash; - mid.receive_headers(&[mid_header]).unwrap(); - let mid_preserved_tip = mid.current_tip_hash; - let mid_preserved_height = mid.current_height; - let mid_preserved_buffered = mid.buffered_headers.len(); - // Simulate a fresh in-flight follow-up request for this segment. - mid.coordinator.mark_sent(&[mid_preserved_tip]); - - let tip_hash = BlockHash::dummy(99); - let mut tip = SegmentState::new(2, 500, tip_hash, None, None); - tip.coordinator.mark_sent(&[tip_hash]); - - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm); - pipeline.initialized = true; - pipeline.next_to_store = 0; - pipeline.segments = vec![completed, mid, tip]; - - pipeline.clear_in_flight(); - - // initialized and next_to_store stay put. - assert!(pipeline.is_initialized()); - assert_eq!(pipeline.next_to_store, 0); - - // Completed segment keeps its buffer and complete flag. - assert!(pipeline.segments[0].complete); - assert_eq!(pipeline.segments[0].buffered_headers.len(), 1); - assert_eq!(pipeline.segments[0].current_tip_hash, shared_hash); - - // Mid-download segment: validated chain state preserved; coordinator wiped. - assert_eq!(pipeline.segments[1].current_tip_hash, mid_preserved_tip); - assert_eq!(pipeline.segments[1].current_height, mid_preserved_height); - assert_eq!(pipeline.segments[1].buffered_headers.len(), mid_preserved_buffered); - assert!(!pipeline.segments[1].complete); - assert_eq!(pipeline.segments[1].coordinator.active_count(), 0); - assert_eq!(pipeline.segments[1].coordinator.pending_count(), 0); - // can_send returns true so a fresh GetHeaders can resume from preserved tip. - assert!(pipeline.segments[1].can_send()); - - // Tip segment: in-flight cleared, preserved hash/height intact. - assert_eq!(pipeline.segments[2].current_tip_hash, tip_hash); - assert_eq!(pipeline.segments[2].coordinator.active_count(), 0); - assert!(pipeline.segments[2].can_send()); - } - - #[test] - fn test_tip_complete_lifecycle() { - let cm = create_test_checkpoint_manager(true); - let mut pipeline = HeadersPipeline::new(cm); - pipeline.initialized = true; - - // No tip segment → not complete - let checkpoint_seg = SegmentState::new(0, 0, BlockHash::dummy(0), Some(100), None); - pipeline.segments = vec![checkpoint_seg]; - assert!(!pipeline.is_tip_complete()); - - // Add a complete tip segment - let mut tip_seg = SegmentState::new(1, 500, BlockHash::dummy(77), None, None); - tip_seg.complete = true; - pipeline.segments.push(tip_seg); - assert!(pipeline.is_tip_complete()); - - // mark_tip_complete is a no-op when already complete - pipeline.mark_tip_complete(); - assert!(pipeline.is_tip_complete()); - - // Simulate receive_headers resetting the tip segment - pipeline.segments[1].complete = false; - assert!(!pipeline.is_tip_complete()); - - // mark_tip_complete restores the state after storing headers - pipeline.mark_tip_complete(); - assert!(pipeline.is_tip_complete()); - } -} diff --git a/dash-spv/src/sync/block_headers/segment_state.rs b/dash-spv/src/sync/block_headers/segment_state.rs index 9829d1250..094f562ec 100644 --- a/dash-spv/src/sync/block_headers/segment_state.rs +++ b/dash-spv/src/sync/block_headers/segment_state.rs @@ -1,12 +1,14 @@ use crate::error::{SyncError, SyncResult}; -use crate::network::RequestSender; -use crate::sync::download_coordinator::{DownloadConfig, DownloadCoordinator}; +use crate::network2::PeerNetworkManager; +use crate::sync::download_coordinator::DownloadCoordinator; use crate::types::HashedBlockHeader; -use dashcore::{BlockHash, Header}; -use std::time::Duration; - -/// Timeout for header requests. -const HEADERS_TIMEOUT: Duration = Duration::from_secs(30); +use dashcore::hashes::Hash; +use dashcore::network::message::NetworkMessage; +use dashcore::network::message_blockdata::GetHeadersMessage; +use dashcore::BlockHash; +#[cfg(test)] +use dashcore::Header; +use std::sync::Arc; /// State for a single download segment between two checkpoints. #[derive(Debug)] @@ -47,11 +49,7 @@ impl SegmentState { target_hash, current_tip_hash: start_hash, current_height: start_height, - coordinator: DownloadCoordinator::new( - DownloadConfig::default() - .with_max_concurrent(1) // Only 1 request at a time (sequential getheaders) - .with_timeout(HEADERS_TIMEOUT), - ), + coordinator: DownloadCoordinator::new(), buffered_headers: Vec::new(), complete: false, } @@ -63,9 +61,32 @@ impl SegmentState { !self.complete && !self.coordinator.is_in_flight(&self.current_tip_hash) } + /// Drop in-flight requests that timed out. Removing the tip hash from the + /// in-flight set makes `can_send` true again, so the next `send_pending` + /// re-issues the GetHeaders to (likely) a different peer. + pub(super) fn handle_timeouts(&mut self, timeout: std::time::Duration) { + let timed_out = self.coordinator.take_timed_out(timeout); + if !timed_out.is_empty() { + tracing::warn!( + "Segment {}: {} request(s) timed out at height {}, will re-send", + self.segment_id, + timed_out.len(), + self.current_height + ); + } + } + /// Send a GetHeaders request for this segment. - pub(super) fn send_request(&mut self, requests: &RequestSender) -> SyncResult<()> { - requests.request_block_headers(self.current_tip_hash)?; + pub(super) async fn send_request( + &mut self, + network: &Arc, + ) -> SyncResult<()> { + network + .send(NetworkMessage::GetHeaders(GetHeadersMessage::new( + vec![self.current_tip_hash], + BlockHash::all_zeros(), + ))) + .await; self.coordinator.mark_sent(&[self.current_tip_hash]); tracing::debug!( "Segment {}: sent GetHeaders from height {} hash {}", @@ -85,8 +106,23 @@ impl SegmentState { /// Process received headers for this segment. /// Returns the number of headers processed, or an error if checkpoint validation fails. + #[cfg(test)] pub(super) fn receive_headers(&mut self, headers: &[Header]) -> SyncResult { - if headers.is_empty() { + let hashed: Vec = headers.iter().map(HashedBlockHeader::from).collect(); + Ok(self + .receive_headers_prehashed(&hashed)? + .map(|(s, e, _)| (e - s + 1) as usize) + .unwrap_or(0)) + } + + /// Route an already-hashed batch to this segment. The expensive X11 block + /// hashing is done in parallel off this task (at the manager, on the tokio + /// runtime), so here we only run the cheap sequential chain/checkpoint check. + pub(super) fn receive_headers_prehashed( + &mut self, + hashed: &[HashedBlockHeader], + ) -> SyncResult> { + if hashed.is_empty() { // Empty response means we've reached the peer's tip for this segment self.complete = true; // Clear in-flight tracking for the current tip hash @@ -96,7 +132,7 @@ impl SegmentState { self.segment_id, self.current_height ); - return Ok(0); + return Ok(None); } // Reject headers on a segment that already reached its checkpoint @@ -104,13 +140,13 @@ impl SegmentState { return Err(SyncError::InvalidState(format!( "Segment {}: received {} headers on completed segment (height {})", self.segment_id, - headers.len(), + hashed.len(), self.current_height ))); } // Mark the request as received, reject if we never requested this hash - let prev_hash = headers[0].prev_blockhash; + let prev_hash = hashed[0].header().prev_blockhash; if !self.coordinator.receive(&prev_hash) { return Err(SyncError::InvalidState(format!( "Segment {}: received unrequested headers (prev_hash {})", @@ -118,11 +154,13 @@ impl SegmentState { ))); } - // Process headers + // Sequentially validate the chain link to the target checkpoint and + // buffer, using the precomputed hashes. + let start_height = self.current_height + 1; let mut processed = 0; - for header in headers { - let hashed = HashedBlockHeader::from(*header); - let hash = *hashed.hash(); + let mut last_hash = self.current_tip_hash; + for h in hashed { + let hash = *h.hash(); let height = self.current_height + processed as u32 + 1; // Check if we've reached the target (next checkpoint) @@ -135,8 +173,9 @@ impl SegmentState { self.segment_id, target_height ); - self.buffered_headers.push(hashed); + self.buffered_headers.push(h.clone()); processed += 1; + last_hash = hash; self.complete = true; break; } else { @@ -155,13 +194,14 @@ impl SegmentState { } } - self.buffered_headers.push(hashed); + self.buffered_headers.push(h.clone()); processed += 1; + last_hash = hash; } // Update current tip for next request if processed > 0 { - self.current_tip_hash = headers[processed - 1].block_hash(); + self.current_tip_hash = last_hash; self.current_height += processed as u32; } @@ -173,7 +213,12 @@ impl SegmentState { self.buffered_headers.len() ); - Ok(processed) + if processed > 0 { + // (start, end, stop_hash) of the batch just validated in memory. + Ok(Some((start_height, self.current_height, last_hash))) + } else { + Ok(None) + } } /// Take buffered headers from this segment. @@ -181,20 +226,6 @@ impl SegmentState { std::mem::take(&mut self.buffered_headers) } - /// Check for timed out requests and handle retries. - pub(super) fn handle_timeouts(&mut self) { - let timed_out = self.coordinator.check_timeouts(); - for hash in timed_out { - tracing::warn!( - "Segment {}: request timed out for hash {}, will retry", - self.segment_id, - hash - ); - // Re-enqueue for retry - self.coordinator.enqueue_retry(hash); - } - } - /// Drop only per-peer in-flight bookkeeping. /// /// Buffered headers and the validated `current_tip_hash` / `current_height` @@ -204,212 +235,3 @@ impl SegmentState { self.coordinator.clear(); } } - -#[cfg(test)] -mod tests { - use crate::error::SyncError; - use crate::sync::block_headers::segment_state::SegmentState; - use crate::types::HashedBlockHeader; - use dashcore::{BlockHash, Header}; - - #[test] - fn test_segment_state_new() { - let hash = BlockHash::dummy(0); - let segment = SegmentState::new(0, 0, hash, Some(500_000), None); - - assert_eq!(segment.segment_id, 0); - assert_eq!(segment.start_height, 0); - assert_eq!(segment.current_height, 0); - assert!(!segment.complete); - assert!(segment.buffered_headers.is_empty()); - } - - #[test] - fn test_segment_can_send() { - let hash = BlockHash::dummy(0); - let segment = SegmentState::new(0, 0, hash, Some(1000), None); - - assert!(segment.can_send()); - } - - #[test] - fn test_segment_matches() { - let hash = BlockHash::dummy(0); - let segment = SegmentState::new(0, 0, hash, Some(1000), None); - - assert!(segment.matches(&hash)); - assert!(!segment.matches(&BlockHash::dummy(1))); - } - - #[test] - fn test_segment_receive_empty() { - let hash = BlockHash::dummy(1); - let mut segment = SegmentState::new(0, 0, hash, Some(1000), None); - - let processed = segment.receive_headers(&[]).unwrap(); - - assert_eq!(processed, 0); - assert!(segment.complete); - } - - #[test] - fn test_segment_receive_headers() { - let hash = BlockHash::dummy(1); - let mut segment = SegmentState::new(0, 0, hash, None, None); - segment.coordinator.mark_sent(&[hash]); - - // Create dummy headers that chain from all-zeros - let headers: Vec
= (1..=10).map(Header::dummy).collect(); - - // Manually fix the prev_blockhash of first header - let mut first = headers[0]; - first.prev_blockhash = hash; - - let processed = segment.receive_headers(&[first]).unwrap(); - - assert_eq!(processed, 1); - assert_eq!(segment.buffered_headers.len(), 1); - assert_eq!(segment.current_height, 1); - assert!(!segment.complete); - } - - #[test] - fn test_segment_checkpoint_mismatch_returns_error() { - let start_hash = BlockHash::dummy(0); - // Segment with checkpoint at height 1 expecting a specific hash - let expected_checkpoint_hash = BlockHash::dummy(99); - let mut segment = - SegmentState::new(0, 0, start_hash, Some(1), Some(expected_checkpoint_hash)); - segment.coordinator.mark_sent(&[start_hash]); - - // Create a header that will be at height 1 but with a different hash - let mut header = Header::dummy(1); - header.prev_blockhash = start_hash; - - // The header's hash won't match the expected checkpoint hash - let hashed = HashedBlockHeader::from(header); - let actual_hash = hashed.hash(); - assert_ne!(*actual_hash, expected_checkpoint_hash); - - // Receiving this header should fail with a validation error - let result = segment.receive_headers(&[header]); - assert!(result.is_err()); - - let err = result.unwrap_err(); - match err { - SyncError::Validation(msg) => { - assert!(msg.contains("does not match checkpoint")); - assert!(msg.contains("height 1")); - } - _ => panic!("Expected SyncError::Validation, got {:?}", err), - } - - // Segment should not be complete and no headers should be buffered - assert!(!segment.complete); - assert!(segment.buffered_headers.is_empty()); - } - - #[test] - fn test_segment_checkpoint_match_completes_segment() { - let start_hash = BlockHash::dummy(0); - // Create a header first to get its hash for the checkpoint - let mut header = Header::dummy(1); - header.prev_blockhash = start_hash; - let hashed = HashedBlockHeader::from(header); - let header_hash = *hashed.hash(); - - // Create segment with checkpoint matching the header's hash - let mut segment = SegmentState::new(0, 0, start_hash, Some(1), Some(header_hash)); - segment.coordinator.mark_sent(&[start_hash]); - - // Receiving this header should succeed and complete the segment - let result = segment.receive_headers(&[header]); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), 1); - - // Segment should be complete with the header buffered - assert!(segment.complete); - assert_eq!(segment.buffered_headers.len(), 1); - } - - #[test] - fn test_unrequested_headers_returns_error() { - let start_hash = BlockHash::dummy(0); - let mut segment = SegmentState::new(0, 0, start_hash, None, None); - - let mut header = Header::dummy(1); - header.prev_blockhash = start_hash; - - let result = segment.receive_headers(&[header]); - assert!(result.is_err()); - match result.unwrap_err() { - SyncError::InvalidState(msg) => { - assert!(msg.contains("unrequested headers")); - } - other => panic!("Expected SyncError::InvalidState, got {:?}", other), - } - assert!(segment.buffered_headers.is_empty()); - } - - #[test] - fn test_completed_segment_rejects_new_headers() { - let start_hash = BlockHash::dummy(0); - let mut segment = SegmentState::new(0, 0, start_hash, Some(100), None); - - // Mark segment as complete (simulating checkpoint reached) - segment.complete = true; - segment.current_height = 100; - - // Create a header that would match - let mut header = Header::dummy(1); - header.prev_blockhash = start_hash; - - // Completed segment should return an invalid state error - let result = segment.receive_headers(&[header]); - assert!(result.is_err()); - match result.unwrap_err() { - SyncError::InvalidState(msg) => { - assert!(msg.contains("completed segment")); - } - other => panic!("Expected SyncError::InvalidState, got {:?}", other), - } - assert!(segment.buffered_headers.is_empty()); - } - - #[test] - fn test_clear_in_flight_preserves_chain_state() { - let start_hash = BlockHash::dummy(0); - let mut segment = SegmentState::new(0, 0, start_hash, None, None); - segment.coordinator.mark_sent(&[start_hash]); - - let mut header = Header::dummy(1); - header.prev_blockhash = start_hash; - segment.receive_headers(&[header]).unwrap(); - - let preserved_tip_hash = segment.current_tip_hash; - let preserved_height = segment.current_height; - let preserved_buffered = segment.buffered_headers.len(); - assert_ne!(preserved_tip_hash, start_hash); - assert_eq!(preserved_height, 1); - assert_eq!(preserved_buffered, 1); - - // Simulate a fresh in-flight request, then clear it. - segment.coordinator.mark_sent(&[preserved_tip_hash]); - assert!(segment.coordinator.is_in_flight(&preserved_tip_hash)); - - segment.clear_in_flight(); - - assert!(!segment.coordinator.is_in_flight(&preserved_tip_hash)); - assert_eq!(segment.coordinator.active_count(), 0); - assert_eq!(segment.coordinator.pending_count(), 0); - - assert_eq!(segment.current_tip_hash, preserved_tip_hash); - assert_eq!(segment.current_height, preserved_height); - assert_eq!(segment.buffered_headers.len(), preserved_buffered); - assert!(!segment.complete); - - // After clearing, can_send should be true again so a fresh GetHeaders - // can resume from the preserved tip hash without re-fetching what we have. - assert!(segment.can_send()); - } -} diff --git a/dash-spv/src/sync/block_headers/sync_manager.rs b/dash-spv/src/sync/block_headers/sync_manager.rs index d47e2cfe8..ba2034499 100644 --- a/dash-spv/src/sync/block_headers/sync_manager.rs +++ b/dash-spv/src/sync/block_headers/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network::{Message, MessageType, NetworkEvent, RequestSender}; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, MetadataStorage}; use crate::sync::sync_manager::ensure_not_started; use crate::sync::{ @@ -9,11 +9,18 @@ use crate::sync::{ use async_trait::async_trait; use dashcore::network::message::NetworkMessage; use dashcore::BlockHash; +use std::net::SocketAddr; +use std::sync::Arc; use std::time::{Duration, Instant}; /// Timeout waiting for unsolicited header messages after a block announcement. pub(super) const UNSOLICITED_HEADERS_WAIT_TIMEOUT: Duration = Duration::from_secs(3); +/// A GetHeaders with no response after this long is assumed lost (the peer died +/// or dropped it) and re-sent. Segments are chained, so one lost request stalls +/// the whole sequential store head until it's retried. +pub(super) const HEADER_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + #[async_trait] impl SyncManager for BlockHeadersManager { fn identifier(&self) -> ManagerIdentifier { @@ -32,10 +39,17 @@ impl SyncManager for BlockHeadersMana self.progress.update_target_height(height); } - fn wanted_message_types(&self) -> &'static [MessageType] { + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { + use crate::network2::MessageType; &[MessageType::Headers, MessageType::Inv] } + fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { + if let crate::network2::RequestKey::Headers(hash) = key { + self.pipeline.mark_on_flight(hash); + } + } + fn on_disconnect(&mut self) { // Drop only per-peer in-flight bookkeeping. Segment topology and // validated chain state per segment (current_tip_hash, current_height, @@ -47,12 +61,24 @@ impl SyncManager for BlockHeadersMana self.announced_peers.clear(); } - async fn start_sync(&mut self, requests: &RequestSender) -> SyncResult> { + async fn start_sync( + &mut self, + network: &Arc, + ) -> SyncResult> { ensure_not_started(self.state(), self.identifier())?; self.progress.set_state(SyncState::Syncing); if !self.pipeline.is_initialized() { let tip = self.tip().await?; + + // Target the highest height advertised by our peers, so the pipeline + // builds segments up to the network tip (parallelism + accurate %). + // Persist it so a restart resumes with the same target. + let peer_tip = network.tip(); + if peer_tip > self.progress.target_height() { + self.progress.update_target_height(peer_tip); + self.metadata_storage.write().await.store_last_target_height(peer_tip).await?; + } let target_height = self.progress.target_height(); // Initialize the pipeline with checkpoint-based segments @@ -78,7 +104,7 @@ impl SyncManager for BlockHeadersMana } // Send initial batch of requests - let sent = self.pipeline.send_pending(requests)?; + let sent = self.pipeline.send_pending(network).await?; tracing::info!("Pipeline: sent {} initial requests", sent); Ok(vec![SyncEvent::SyncStart { @@ -88,17 +114,18 @@ impl SyncManager for BlockHeadersMana async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + _peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult> { - match msg.inner() { + match &msg { NetworkMessage::Headers(headers) => { // Always route through pipeline when initialized - self.handle_headers_pipeline(headers, requests).await + self.handle_headers_pipeline(headers, network).await } NetworkMessage::Inv(inv) => { - self.handle_inventory(inv, requests).await?; + self.handle_inventory(inv, network).await?; Ok(vec![]) } @@ -109,22 +136,21 @@ impl SyncManager for BlockHeadersMana async fn handle_sync_event( &mut self, _event: &SyncEvent, - _requests: &RequestSender, + _network: &Arc, ) -> SyncResult> { // BlockHeadersManager doesn't react to events from other managers Ok(vec![]) } - async fn tick(&mut self, requests: &RequestSender) -> SyncResult> { + async fn tick(&mut self, network: &Arc) -> SyncResult> { if !self.pipeline.is_initialized() { return Ok(vec![]); } - self.pipeline.handle_timeouts(); - - // During initial sync, send more requests and log progress + // During initial sync, re-send any timed-out requests then push pending. if self.state() == SyncState::Syncing { - let sent = self.pipeline.send_pending(requests)?; + self.pipeline.handle_timeouts(HEADER_REQUEST_TIMEOUT); + let sent = self.pipeline.send_pending(network).await?; if sent > 0 { tracing::debug!("Tick: pipeline sent {} more requests", sent); } @@ -152,7 +178,7 @@ impl SyncManager for BlockHeadersMana // Reset tip segment and send requests via pipeline self.pipeline.reset_tip_segment(); - self.pipeline.send_pending(requests)?; + self.pipeline.send_pending(network).await?; for hash in stale { self.pending_announcements.remove(&hash); @@ -163,74 +189,6 @@ impl SyncManager for BlockHeadersMana Ok(vec![]) } - async fn handle_network_event( - &mut self, - event: &NetworkEvent, - requests: &RequestSender, - ) -> SyncResult> { - match event { - NetworkEvent::PeerConnected { - address, - } => { - // When synced, send GetHeaders to new peers so Dash Core learns our tip - // and sends header announcements instead of inv. Skip when the - // pipeline has an active catch-up request to avoid the empty - // response prematurely completing the tip segment. - if self.state() == SyncState::Synced - && self.pipeline.is_initialized() - && !self.announced_peers.contains(address) - && !self.pipeline.tip_segment_has_pending_request() - { - let tip = self.tip().await?; - tracing::info!("Announcing tip {} to new peer {}", tip.height(), address); - requests.request_block_headers_from_peer(*tip.hash(), *address)?; - self.announced_peers.insert(*address); - } - } - NetworkEvent::PeerDisconnected { - address, - } => { - self.announced_peers.remove(address); - } - NetworkEvent::PeersUpdated { - connected_count, - best_height, - .. - } => { - if let Some(best_height) = best_height { - self.progress.update_target_height(*best_height); - let mut metadata_storage = self.metadata_storage.write().await; - metadata_storage.store_last_target_height(*best_height).await?; - } - if *connected_count == 0 { - self.stop_sync(); - } else if *connected_count > 0 { - if self.state() == SyncState::WaitingForConnections { - return self.start_sync(requests).await; - } - // When already synced but behind peer height, request missing headers - if self.state() == SyncState::Synced { - if let Some(best_height) = best_height { - if *best_height > self.progress.tip_height() - && !self.pipeline.tip_segment_has_pending_request() - { - tracing::info!( - "Peer height {} > our height {}, requesting headers to catch up", - best_height, - self.progress.tip_height() - ); - // Reset tip segment and send requests via pipeline - self.pipeline.reset_tip_segment(); - self.pipeline.send_pending(requests)?; - } - } - } - } - } - } - Ok(vec![]) - } - fn progress(&self) -> SyncManagerProgress { let mut progress = self.progress.clone(); progress.update_buffered(self.pipeline.total_buffered()); diff --git a/dash-spv/src/sync/blocks/manager.rs b/dash-spv/src/sync/blocks/manager.rs index 70df8624b..47538f06d 100644 --- a/dash-spv/src/sync/blocks/manager.rs +++ b/dash-spv/src/sync/blocks/manager.rs @@ -9,7 +9,7 @@ use tokio::sync::RwLock; use super::pipeline::BlocksPipeline; use crate::error::SyncResult; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, BlockStorage}; use crate::sync::{BlocksProgress, SyncEvent, SyncManager, SyncState}; use key_wallet_manager::WalletInterface; @@ -63,8 +63,11 @@ impl BlocksManager SyncResult<()> { - let sent = self.pipeline.send_pending(requests).await?; + pub(super) async fn send_pending( + &mut self, + network: &Arc, + ) -> SyncResult<()> { + let sent = self.pipeline.send_pending(network).await?; if sent > 0 { self.progress.add_requested(sent as u32); } @@ -163,219 +166,3 @@ impl std::fmt::Debug .finish() } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::network::{MessageType, NetworkManager}; - use crate::storage::{ - DiskStorageManager, PersistentBlockHeaderStorage, PersistentBlockStorage, StorageManager, - }; - use crate::sync::{ManagerIdentifier, SyncEvent, SyncManagerProgress}; - use crate::test_utils::MockNetworkManager; - use crate::types::HashedBlock; - use key_wallet_manager::test_utils::{MockWallet, MOCK_WALLET_ID}; - use key_wallet_manager::FilterMatchKey; - use std::collections::{BTreeMap, BTreeSet}; - - type TestBlocksManager = - BlocksManager; - type TestSyncManager = dyn SyncManager; - - async fn create_test_manager() -> TestBlocksManager { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let wallet = Arc::new(RwLock::new(MockWallet::new())); - BlocksManager::new(wallet, storage.block_headers(), storage.blocks()).await - } - - #[tokio::test] - async fn test_blocks_manager_new() { - let manager = create_test_manager().await; - assert_eq!(manager.identifier(), ManagerIdentifier::Block); - assert_eq!(manager.state(), SyncState::WaitForEvents); - assert_eq!(manager.wanted_message_types(), vec![MessageType::Block]); - } - - #[tokio::test] - async fn test_blocks_manager_progress() { - let mut manager = create_test_manager().await; - manager.progress.update_last_processed(500); - manager.progress.add_processed(10); - - let manager_ref: &TestSyncManager = &manager; - let progress = manager_ref.progress(); - if let SyncManagerProgress::Blocks(blocks_progress) = progress { - assert_eq!(blocks_progress.last_processed(), 500); - assert_eq!(blocks_progress.processed(), 10); - } else { - panic!("Expected SyncManagerProgress::Blocks"); - } - } - - #[tokio::test] - async fn test_blocks_manager_handle_blocks_needed_event() { - let mut manager = create_test_manager().await; - manager.progress.set_state(SyncState::Synced); - - let network = MockNetworkManager::new(); - let requests = network.request_sender(); - - let block_hash = dashcore::BlockHash::dummy(0); - let mut blocks = BTreeMap::new(); - blocks.insert(FilterMatchKey::new(100, block_hash), BTreeSet::from([MOCK_WALLET_ID])); - let event = SyncEvent::BlocksNeeded { - blocks, - }; - - let events = manager.handle_sync_event(&event, &requests).await.unwrap(); - - // Should queue the block - assert_eq!(manager.state(), SyncState::Syncing); - assert!(events.is_empty()); - } - - /// `process_buffered_blocks` must call `process_block_for_wallets` with - /// the exact wallet set carried in the pipeline so already-synced - /// wallets are not touched by routing logic. - #[tokio::test] - async fn test_process_buffered_blocks_routes_wallet_set() { - use dashcore::block::Header; - use dashcore::{Block, TxMerkleNode}; - use dashcore_hashes::Hash; - - let mut manager = create_test_manager().await; - manager.progress.set_state(SyncState::Syncing); - - let header = Header { - version: dashcore::blockdata::block::Version::from_consensus(1), - prev_blockhash: dashcore::BlockHash::all_zeros(), - merkle_root: TxMerkleNode::all_zeros(), - time: 0, - bits: dashcore::CompactTarget::from_consensus(0), - nonce: 0, - }; - let block = Block { - header, - txdata: vec![], - }; - manager.pipeline.add_from_storage( - HashedBlock::from(&block), - 100, - BTreeSet::from([MOCK_WALLET_ID]), - ); - - let events = manager.process_buffered_blocks().await.unwrap(); - assert!(matches!(events.first(), Some(SyncEvent::BlockProcessed { .. }))); - - // MOCK_WALLET_ID was in the routed set, so MockWallet recorded the - // block. (MockWallet::process_block_for_wallets returns early when - // its id is absent.) - let processed = manager.wallet.read().await.processed_blocks(); - let processed = processed.lock().await; - assert_eq!(processed.len(), 1); - assert_eq!(processed[0].1, 100); - } - - /// A wallet that is NOT in the pipeline's interested set must not be - /// routed the block. Two wallets are registered, but only `wallet_in` - /// appears in the routed set; the other wallet's processed log must - /// stay empty for that block. - #[tokio::test] - async fn test_process_buffered_blocks_excludes_uninterested_wallet() { - use dashcore::block::Header; - use dashcore::{Block, TxMerkleNode}; - use dashcore_hashes::Hash; - use key_wallet_manager::test_utils::{MockWalletState, MultiMockWallet}; - use key_wallet_manager::WalletId; - - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let multi = MultiMockWallet::new(); - let wallet_in: WalletId = [0xAA; 32]; - let wallet_out: WalletId = [0xBB; 32]; - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet(wallet_in, MockWalletState::default()); - w.insert_wallet(wallet_out, MockWalletState::default()); - } - let mut manager: BlocksManager< - PersistentBlockHeaderStorage, - PersistentBlockStorage, - MultiMockWallet, - > = BlocksManager::new(multi.clone(), storage.block_headers(), storage.blocks()).await; - manager.progress.set_state(SyncState::Syncing); - - let header = Header { - version: dashcore::blockdata::block::Version::from_consensus(1), - prev_blockhash: dashcore::BlockHash::all_zeros(), - merkle_root: TxMerkleNode::all_zeros(), - time: 0, - bits: dashcore::CompactTarget::from_consensus(0), - nonce: 0, - }; - let block = Block { - header, - txdata: vec![], - }; - // Only wallet_in is in the routed set. - manager.pipeline.add_from_storage( - HashedBlock::from(&block), - 100, - BTreeSet::from([wallet_in]), - ); - - let _ = manager.process_buffered_blocks().await.unwrap(); - - let processed = multi.read().await.processed(); - let processed = processed.lock().await; - // Exactly one entry, for wallet_in only. - assert_eq!(processed.len(), 1); - assert_eq!(processed[0].0, wallet_in); - assert_eq!(processed[0].2, 100); - assert!( - !processed.iter().any(|(id, _, _)| *id == wallet_out), - "wallet_out was not in the routed set, must not be processed" - ); - } - - /// `on_disconnect` for `BlocksManager` keeps the downloaded buffer, the - /// per-block wallet routing, and the `filters_sync_complete` flag, and - /// moves any in-flight `getdata`s back to pending so the next - /// `send_pending` reissues them. Without this preservation, blocks waiting - /// in `downloaded` for height ordering would be dropped, leaving - /// `FiltersManager.tracker` entries that never get decremented. - #[tokio::test] - async fn test_on_disconnect_preserves_pipeline_work() { - use dashcore::block::Header; - use dashcore::{Block, TxMerkleNode}; - use dashcore_hashes::Hash; - - let mut manager = create_test_manager().await; - manager.filters_sync_complete = true; - - let header = Header { - version: dashcore::blockdata::block::Version::from_consensus(1), - prev_blockhash: dashcore::BlockHash::all_zeros(), - merkle_root: TxMerkleNode::all_zeros(), - time: 0, - bits: dashcore::CompactTarget::from_consensus(0), - nonce: 0, - }; - let block = Block { - header, - txdata: vec![], - }; - - // Already-downloaded block sitting in the pipeline. - manager.pipeline.add_from_storage( - HashedBlock::from(&block), - 200, - BTreeSet::from([MOCK_WALLET_ID]), - ); - - manager.on_disconnect(); - - assert!(!manager.pipeline.is_complete(), "downloaded buffer must survive on_disconnect"); - assert!(manager.filters_sync_complete); - } -} diff --git a/dash-spv/src/sync/blocks/pipeline.rs b/dash-spv/src/sync/blocks/pipeline.rs index 02b29aac2..2c684d73b 100644 --- a/dash-spv/src/sync/blocks/pipeline.rs +++ b/dash-spv/src/sync/blocks/pipeline.rs @@ -1,34 +1,25 @@ //! Blocks pipeline implementation. //! -//! Handles concurrent block downloads with timeout and retry logic. -//! Uses the generic DownloadCoordinator for core mechanics. +//! Handles concurrent block downloads. use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::time::Duration; +use std::sync::Arc; use crate::error::SyncResult; -use crate::network::RequestSender; -use crate::sync::download_coordinator::{DownloadConfig, DownloadCoordinator}; +use crate::network2::PeerNetworkManager; +use crate::sync::download_coordinator::DownloadCoordinator; use crate::types::HashedBlock; +use dashcore::network::message::NetworkMessage; +use dashcore::network::message_blockdata::Inventory; use dashcore::BlockHash; use key_wallet_manager::{FilterMatchKey, WalletId}; -/// Maximum number of concurrent block downloads. -const MAX_CONCURRENT_BLOCK_DOWNLOADS: usize = 20; - -/// Timeout for block downloads before retry. -const BLOCK_TIMEOUT: Duration = Duration::from_secs(30); - -/// Maximum blocks per GetData request, kept a bit lower for better download distribution to multiple peers -const BLOCKS_PER_REQUEST: usize = 8; - /// Pipeline for downloading blocks with height-ordered processing. /// -/// Uses DownloadCoordinator for core download mechanics. /// This is a thin wrapper that handles building GetData inventory messages. /// Tracks block heights to enable ordered processing and buffers downloaded blocks. pub(super) struct BlocksPipeline { - /// Core download coordinator (handles pending, in-flight, timeouts). + /// Coordinates pending/in-flight block hashes for download. coordinator: DownloadCoordinator, /// Heights queued or in-flight (waiting for download). pending_heights: BTreeSet, @@ -61,11 +52,7 @@ impl BlocksPipeline { /// Create a new blocks pipeline. pub(super) fn new() -> Self { Self { - coordinator: DownloadCoordinator::new( - DownloadConfig::default() - .with_max_concurrent(MAX_CONCURRENT_BLOCK_DOWNLOADS) - .with_timeout(BLOCK_TIMEOUT), - ), + coordinator: DownloadCoordinator::new(), pending_heights: BTreeSet::new(), downloaded: BTreeMap::new(), hash_to_height: HashMap::new(), @@ -103,53 +90,95 @@ impl BlocksPipeline { self.coordinator.available_to_send() > 0 } - /// Send pending block requests up to the concurrency limit. + /// Start the response timeout for a block the router just put on the wire. + pub(super) fn mark_on_flight(&mut self, hash: &BlockHash) { + self.coordinator.mark_on_flight(std::slice::from_ref(hash)); + } + + /// Re-queue block downloads whose response never arrived (peer died or the + /// block was lost), so the next `send_pending` re-issues the GetData. + pub(super) fn handle_timeouts(&mut self, timeout: std::time::Duration) { + let timed_out = self.coordinator.take_timed_out(timeout); + if !timed_out.is_empty() { + tracing::warn!("Blocks: re-queuing {} timed-out block(s)", timed_out.len()); + self.coordinator.enqueue(timed_out); + } + } + + /// Offer every pending block to the network manager. + /// + /// Blocks go through the router like every other request — one `getdata` per + /// block, so a request is one in-flight unit on the peer and one `block` in + /// reply, and the peer's measured capacity throttles them. (They used to bypass + /// the router entirely, a direct-send from the days when the single FIFO left + /// them stuck behind hundreds of `getcfilters`. The class-based queue fixed that + /// starvation properly; unpaced, the bypass then buried peers under thousands of + /// 2MB requests, which they silently dropped, hanging the sync on blocks that + /// were never served.) /// - /// Sends multiple smaller GetData messages to distribute requests across peers. /// Returns the number of blocks requested. - pub(super) async fn send_pending(&mut self, requests: &RequestSender) -> SyncResult { - let mut total_sent = 0; + pub(super) async fn send_pending( + &mut self, + network: &Arc, + ) -> SyncResult { + let hashes = self.coordinator.take_pending(self.coordinator.available_to_send()); + if hashes.is_empty() { + return Ok(0); + } - while self.coordinator.available_to_send() > 0 { - // Take a batch of up to BLOCKS_PER_REQUEST - let count = self.coordinator.available_to_send().min(BLOCKS_PER_REQUEST); - let hashes = self.coordinator.take_pending(count); - if hashes.is_empty() { - break; - } + for hash in &hashes { + network.send(NetworkMessage::GetData(vec![Inventory::Block(*hash)])).await; + } + // Handed to the network; the response timeout starts when the router reports + // each one actually on the wire (`RequestKey::Block` -> `mark_on_flight`). + self.coordinator.mark_sent(&hashes); - requests.request_blocks(hashes.clone())?; - self.coordinator.mark_sent(&hashes); - total_sent += hashes.len(); + tracing::debug!( + "Requested {} blocks ({} in flight, {} pending)", + hashes.len(), + self.coordinator.active_count(), + self.coordinator.pending_count() + ); - tracing::debug!( - "Requested {} blocks ({} downloading, {} pending)", - hashes.len(), - self.coordinator.active_count(), - self.coordinator.pending_count() - ); - } + Ok(hashes.len()) + } + + /// Bytes of blocks downloaded and waiting in RAM for their turn to be processed + /// (the wallet must see them in height order). Up to ~2MB each — the reason this + /// buffer is worth watching. + pub(super) fn buffered_bytes(&self) -> usize { + self.downloaded + .values() + .map(|b| dashcore::consensus::encode::serialize(b.block()).len()) + .sum() + } - Ok(total_sent) + /// Number of blocks downloaded and waiting to be processed. + pub(super) fn buffered_blocks(&self) -> usize { + self.downloaded.len() } /// Handle a received block using internal height mapping. /// - /// Looks up the height from the internal hash_to_height map and stores - /// the block in the downloaded buffer for height-ordered processing. - /// Returns `true` if this was a tracked block, `false` if unrequested. - pub(super) fn receive_block(&mut self, block: &HashedBlock) -> bool { + /// Stores the block in the downloaded buffer for height-ordered processing and + /// returns **the height it was queued at**, or `None` if the block was never + /// requested. `queue` records the height alongside the hash, so the caller gets it + /// from here instead of asking the storage's chain-wide hash index to recover what + /// this pipeline already knew. + pub(super) fn receive_block(&mut self, block: &HashedBlock) -> Option { let hash = *block.hash(); if !self.coordinator.receive(&hash) { tracing::debug!("Ignoring unrequested block: {}", hash); - return false; + return None; } - if let Some(height) = self.hash_to_height.remove(&hash) { - self.pending_heights.remove(&height); - self.downloaded.insert(height, block.clone()); - } - true + // `queue` populates the coordinator and `hash_to_height` together, so a block + // the coordinator accepted always has a height here. + let height = self.hash_to_height.remove(&hash)?; + self.pending_heights.remove(&height); + self.downloaded.insert(height, block.clone()); + + Some(height) } /// Take the next block that's safe to process in height order, along with @@ -188,446 +217,4 @@ impl BlocksPipeline { self.hash_to_wallets.entry(hash).or_default().extend(wallets); self.downloaded.insert(height, block); } - - /// Check for timed out downloads and re-queue them. - pub(super) fn handle_timeouts(&mut self) { - self.coordinator.check_and_retry_timeouts(); - } - - /// Move in-flight `getdata` requests back to pending after a peer - /// disconnect so the next `send_pending` reissues them to the new peer. - /// `pending_heights`, `downloaded`, `hash_to_height`, and `hash_to_wallets` - /// are preserved so already-received blocks are not re-fetched and the - /// per-block wallet routing stays intact. - pub(super) fn requeue_in_flight(&mut self) { - self.coordinator.requeue_in_flight(); - } -} - -#[cfg(test)] -mod tests { - use dashcore::blockdata::block::Block; - use dashcore_hashes::Hash; - - use super::*; - - fn test_hash(n: u8) -> BlockHash { - BlockHash::from_byte_array([n; 32]) - } - - fn make_test_block(n: u8) -> Block { - use dashcore::blockdata::block::Header; - let header = Header { - version: dashcore::blockdata::block::Version::from_consensus(1), - prev_blockhash: BlockHash::from_byte_array([n; 32]), - merkle_root: dashcore::TxMerkleNode::all_zeros(), - time: n as u32, - bits: dashcore::CompactTarget::from_consensus(0), - nonce: n as u32, - }; - Block { - header, - txdata: vec![], - } - } - - #[test] - fn test_blocks_pipeline_new() { - let pipeline = BlocksPipeline::new(); - assert_eq!(pipeline.coordinator.pending_count(), 0); - assert_eq!(pipeline.coordinator.active_count(), 0); - assert!(pipeline.is_complete()); - } - - #[test] - fn test_queue_block() { - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - pipeline.queue([(FilterMatchKey::new(100, block.block_hash()), BTreeSet::new())]); - - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert!(!pipeline.is_complete()); - assert!(pipeline.has_pending_requests()); - } - - #[test] - fn test_queue_multiple() { - let mut pipeline = BlocksPipeline::new(); - let block1 = make_test_block(1); - let block2 = make_test_block(2); - let block3 = make_test_block(3); - pipeline.queue([ - (FilterMatchKey::new(100, block1.block_hash()), BTreeSet::new()), - (FilterMatchKey::new(101, block2.block_hash()), BTreeSet::new()), - (FilterMatchKey::new(102, block3.block_hash()), BTreeSet::new()), - ]); - - assert_eq!(pipeline.coordinator.pending_count(), 3); - assert_eq!(pipeline.pending_heights.len(), 3); - assert!(pipeline.pending_heights.contains(&100)); - assert!(pipeline.pending_heights.contains(&101)); - assert!(pipeline.pending_heights.contains(&102)); - } - - #[test] - fn test_receive_block_with_height() { - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - let hash = block.block_hash(); - - // Queue with height tracking - pipeline.queue([(FilterMatchKey::new(100, block.block_hash()), BTreeSet::new())]); - - // Simulate sending via coordinator - let hashes = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&hashes); - assert_eq!(pipeline.coordinator.active_count(), 1); - - // Receive block - assert!(pipeline.receive_block(&HashedBlock::from(&block))); - assert_eq!(pipeline.coordinator.active_count(), 0); - assert_eq!(pipeline.downloaded.len(), 1); - assert!(pipeline.pending_heights.is_empty()); - assert_eq!(*pipeline.downloaded.get(&100).unwrap().hash(), hash); - } - - #[test] - fn test_receive_block_unrequested() { - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - - assert!(!pipeline.receive_block(&HashedBlock::from(&block))); - assert!(pipeline.downloaded.is_empty()); - } - - #[test] - fn test_max_concurrent() { - let mut pipeline = BlocksPipeline::new(); - - // Queue more blocks than max concurrent - for i in 0..=MAX_CONCURRENT_BLOCK_DOWNLOADS { - let block = make_test_block(i as u8); - pipeline.queue([(FilterMatchKey::new(i as u32, block.block_hash()), BTreeSet::new())]); - } - - // Take and mark as downloading up to limit - let to_send = pipeline.coordinator.available_to_send(); - let hashes = pipeline.coordinator.take_pending(to_send); - pipeline.coordinator.mark_sent(&hashes); - - assert_eq!(pipeline.coordinator.active_count(), MAX_CONCURRENT_BLOCK_DOWNLOADS); - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert!(!pipeline.has_pending_requests()); - } - - #[test] - fn test_requeue_in_flight_preserves_downloaded_and_pending_heights() { - let mut pipeline = BlocksPipeline::new(); - let block_a = make_test_block(1); - let block_b = make_test_block(2); - let hash_a = block_a.block_hash(); - let hash_b = block_b.block_hash(); - - // A: queued and sent — will be requeued. - pipeline.queue([(FilterMatchKey::new(100, hash_a), BTreeSet::from([[1u8; 32]]))]); - let sent = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&sent); - assert_eq!(pipeline.coordinator.active_count(), 1); - - // B: already received, sitting in `downloaded` — must survive requeue. - pipeline.add_from_storage(HashedBlock::from(&block_b), 200, BTreeSet::from([[2u8; 32]])); - - pipeline.requeue_in_flight(); - - assert_eq!(pipeline.coordinator.active_count(), 0); - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert!(pipeline.pending_heights.contains(&100)); - assert_eq!(pipeline.hash_to_height.get(&hash_a), Some(&100)); - assert!(pipeline.hash_to_wallets.contains_key(&hash_a)); - assert!(pipeline.downloaded.contains_key(&200)); - assert!(pipeline.hash_to_wallets.contains_key(&hash_b)); - } - - #[test] - fn test_timeout_requeues() { - // Create pipeline with very short timeout for testing - let mut pipeline = BlocksPipeline { - coordinator: DownloadCoordinator::new( - DownloadConfig::default() - .with_max_concurrent(MAX_CONCURRENT_BLOCK_DOWNLOADS) - .with_timeout(Duration::from_millis(10)), - ), - pending_heights: BTreeSet::new(), - downloaded: BTreeMap::new(), - hash_to_height: HashMap::new(), - hash_to_wallets: HashMap::new(), - }; - - // Use coordinator directly to set up in-flight state - let hash = test_hash(1); - pipeline.coordinator.enqueue([hash]); - let hashes = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&hashes); - - // Wait for timeout - std::thread::sleep(Duration::from_millis(20)); - - pipeline.handle_timeouts(); - - assert_eq!(pipeline.coordinator.active_count(), 0); - assert_eq!(pipeline.coordinator.pending_count(), 1); - } - - #[test] - fn test_take_next_ordered_block_in_order() { - let mut pipeline = BlocksPipeline::new(); - let block1 = make_test_block(1); - let block2 = make_test_block(2); - let hash1 = block1.block_hash(); - let hash2 = block2.block_hash(); - - // Use add_from_storage to test ordering logic without network - // Add block 2 first (out of order) - pipeline.add_from_storage(HashedBlock::from(&block2), 101, BTreeSet::new()); - // Also track height 100 as pending to simulate waiting - pipeline.pending_heights.insert(100); - - // Cannot take block 2 yet - waiting for block at height 100 - assert!(pipeline.take_next_ordered_block().is_none()); - - // Add block 1 - pipeline.pending_heights.remove(&100); - pipeline.add_from_storage(HashedBlock::from(&block1), 100, BTreeSet::new()); - - // Now block 1 is ready (lowest height) - let (block, height, _) = pipeline.take_next_ordered_block().unwrap(); - assert_eq!(height, 100); - assert_eq!(*block.hash(), hash1); - - // Block 2 is now ready - let (block, height, _) = pipeline.take_next_ordered_block().unwrap(); - assert_eq!(height, 101); - assert_eq!(*block.hash(), hash2); - - // No more blocks - assert!(pipeline.take_next_ordered_block().is_none()); - } - - #[test] - fn test_take_next_ordered_block_waits_for_pending() { - let mut pipeline = BlocksPipeline::new(); - let block2 = make_test_block(2); - - // Add block at height 101, but height 100 is still pending - pipeline.pending_heights.insert(100); - pipeline.add_from_storage(HashedBlock::from(&block2), 101, BTreeSet::new()); - - // Cannot take block 2 - block at height 100 is still pending - assert!(pipeline.take_next_ordered_block().is_none()); - - // Clear the pending height - pipeline.pending_heights.remove(&100); - - // Now block 2 is ready - let (_, height, _) = pipeline.take_next_ordered_block().unwrap(); - assert_eq!(height, 101); - } - - #[test] - fn test_add_from_storage() { - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - let hash = block.block_hash(); - - pipeline.add_from_storage(HashedBlock::from(&block), 100, BTreeSet::new()); - - assert_eq!(pipeline.downloaded.len(), 1); - - let (taken_block, height, _) = pipeline.take_next_ordered_block().unwrap(); - assert_eq!(height, 100); - assert_eq!(*taken_block.hash(), hash); - } - - #[test] - fn test_is_complete() { - let mut pipeline = BlocksPipeline::new(); - assert!(pipeline.is_complete()); - - // Adding to downloaded makes it incomplete - let block = make_test_block(1); - pipeline.add_from_storage(HashedBlock::from(&block), 100, BTreeSet::new()); - assert!(!pipeline.is_complete()); - - // Take the block - pipeline.take_next_ordered_block(); - assert!(pipeline.is_complete()); - } - - #[test] - fn test_is_complete_with_pending_heights() { - let mut pipeline = BlocksPipeline::new(); - assert!(pipeline.is_complete()); - - // Pending heights make it incomplete - pipeline.pending_heights.insert(100); - assert!(!pipeline.is_complete()); - - pipeline.pending_heights.remove(&100); - assert!(pipeline.is_complete()); - } - - #[test] - fn test_queue_propagates_wallet_set_through_take_next() { - // A block queued with a non-empty wallet set must yield that exact - // wallet set when taken in height order via `take_next_ordered_block`. - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - let hash = block.block_hash(); - let wallets: BTreeSet = BTreeSet::from([[1u8; 32], [2u8; 32]]); - - pipeline.queue([(FilterMatchKey::new(100, hash), wallets.clone())]); - - // Drive the block through receive_block to land it in `downloaded`. - let hashes = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&hashes); - assert!(pipeline.receive_block(&HashedBlock::from(&block))); - - let (taken_block, height, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); - assert_eq!(*taken_block.hash(), hash); - assert_eq!(height, 100); - assert_eq!(taken_wallets, wallets); - } - - #[test] - fn test_queue_merges_wallet_sets_for_repeat_hashes() { - // Queueing the same block hash twice with different wallet sets must - // produce the union when the block is later taken from the pipeline, - // and must not double-count it in the coordinator's pending state. - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - let hash = block.block_hash(); - let wallets_a: BTreeSet = BTreeSet::from([[1u8; 32]]); - let wallets_b: BTreeSet = BTreeSet::from([[2u8; 32], [3u8; 32]]); - - pipeline.queue([(FilterMatchKey::new(100, hash), wallets_a.clone())]); - assert_eq!(pipeline.coordinator.pending_count(), 1); - pipeline.queue([(FilterMatchKey::new(100, hash), wallets_b.clone())]); - // Re-queueing must not double the coordinator's pending count. - assert_eq!(pipeline.coordinator.pending_count(), 1); - - // Land the block in `downloaded` to retrieve it. - let hashes = pipeline.coordinator.take_pending(1); - assert_eq!(hashes.len(), 1); - pipeline.coordinator.mark_sent(&hashes); - assert!(pipeline.receive_block(&HashedBlock::from(&block))); - - let (_, _, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); - let mut expected = wallets_a; - expected.extend(wallets_b); - assert_eq!(taken_wallets, expected); - } - - #[test] - fn test_queue_does_not_re_enqueue_in_flight_hash() { - // A late-arriving wallet match for a block already in flight must - // merge the wallet id without re-enqueueing the hash. Re-enqueueing - // would cause a duplicate request and corrupt the coordinator's - // pending/in-flight state. - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - let hash = block.block_hash(); - let wallets_a: BTreeSet = BTreeSet::from([[1u8; 32]]); - let wallets_b: BTreeSet = BTreeSet::from([[2u8; 32]]); - - pipeline.queue([(FilterMatchKey::new(100, hash), wallets_a.clone())]); - // Move the hash to in-flight. - let hashes = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&hashes); - assert_eq!(pipeline.coordinator.pending_count(), 0); - assert_eq!(pipeline.coordinator.active_count(), 1); - - // A second queue call for the same hash must not push it back to - // pending while it is in flight. - pipeline.queue([(FilterMatchKey::new(100, hash), wallets_b.clone())]); - assert_eq!(pipeline.coordinator.pending_count(), 0); - assert_eq!(pipeline.coordinator.active_count(), 1); - - // Late wallet ids are still merged for when the block arrives. - assert!(pipeline.receive_block(&HashedBlock::from(&block))); - let (_, _, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); - let mut expected = wallets_a; - expected.extend(wallets_b); - assert_eq!(taken_wallets, expected); - } - - #[test] - fn test_queue_does_not_re_enqueue_downloaded_hash() { - // A late-arriving wallet match for a block already received and sitting - // in `downloaded` (but not yet consumed by `take_next_ordered_block`) - // must merge the wallet id without re-enqueueing the hash. - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - let hash = block.block_hash(); - let wallets_a: BTreeSet = BTreeSet::from([[1u8; 32]]); - let wallets_b: BTreeSet = BTreeSet::from([[2u8; 32]]); - - pipeline.queue([(FilterMatchKey::new(100, hash), wallets_a.clone())]); - let hashes = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&hashes); - assert!(pipeline.receive_block(&HashedBlock::from(&block))); - assert_eq!(pipeline.downloaded.len(), 1); - assert_eq!(pipeline.coordinator.pending_count(), 0); - assert_eq!(pipeline.coordinator.active_count(), 0); - - // Late-arriving match for the same hash must not re-enqueue. - pipeline.queue([(FilterMatchKey::new(100, hash), wallets_b.clone())]); - assert_eq!(pipeline.coordinator.pending_count(), 0); - assert_eq!(pipeline.coordinator.active_count(), 0); - assert_eq!(pipeline.downloaded.len(), 1); - - // Late wallet ids are still merged for when the block is taken. - let (_, _, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); - let mut expected = wallets_a; - expected.extend(wallets_b); - assert_eq!(taken_wallets, expected); - } - - #[test] - fn test_add_from_storage_merges_wallet_sets() { - // The `add_from_storage` path must merge wallet sets for repeat - // additions of the same block hash, matching `queue`'s semantics. - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - let wallets_a: BTreeSet = BTreeSet::from([[1u8; 32]]); - let wallets_b: BTreeSet = BTreeSet::from([[2u8; 32]]); - - pipeline.add_from_storage(HashedBlock::from(&block), 100, wallets_a.clone()); - pipeline.add_from_storage(HashedBlock::from(&block), 100, wallets_b.clone()); - - let (_, _, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); - let mut expected = wallets_a; - expected.extend(wallets_b); - assert_eq!(taken_wallets, expected); - } - - #[test] - fn test_receive_block_duplicate() { - let mut pipeline = BlocksPipeline::new(); - let block = make_test_block(1); - - // Queue and mark as sent via coordinator - pipeline.queue([(FilterMatchKey::new(100, block.block_hash()), BTreeSet::new())]); - let hashes = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&hashes); - - // First receive - let result = pipeline.receive_block(&HashedBlock::from(&block)); - assert!(result); - assert_eq!(pipeline.downloaded.len(), 1); - - // Duplicate receive (not tracked anymore since already completed) - let result = pipeline.receive_block(&HashedBlock::from(&block)); - assert!(!result); - assert_eq!(pipeline.downloaded.len(), 1); - } } diff --git a/dash-spv/src/sync/blocks/sync_manager.rs b/dash-spv/src/sync/blocks/sync_manager.rs index e7ecbc68a..0f7be55bc 100644 --- a/dash-spv/src/sync/blocks/sync_manager.rs +++ b/dash-spv/src/sync/blocks/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network::{Message, MessageType, RequestSender}; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, BlockStorage}; use crate::sync::sync_manager::ensure_not_started; use crate::sync::{ @@ -11,6 +11,8 @@ use async_trait::async_trait; use dashcore::network::message::NetworkMessage; use key_wallet_manager::{FilterMatchKey, WalletId, WalletInterface}; use std::collections::BTreeSet; +use std::net::SocketAddr; +use std::sync::Arc; #[async_trait] impl SyncManager @@ -28,11 +30,22 @@ impl SyncM self.progress.set_state(state); } - fn wanted_message_types(&self) -> &'static [MessageType] { - &[MessageType::Block] + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { + &[crate::network2::MessageType::Block] } - async fn start_sync(&mut self, _requests: &RequestSender) -> SyncResult> { + /// The router put a block request on the wire: start ITS response timeout from + /// here, not from when it was queued — the queue can hold it for a while. + fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { + if let crate::network2::RequestKey::Block(hash) = key { + self.pipeline.mark_on_flight(hash); + } + } + + async fn start_sync( + &mut self, + _network: &Arc, + ) -> SyncResult> { ensure_not_started(self.state(), self.identifier())?; // Check if filters already completed (event received before start_sync) if self.filters_sync_complete && self.pipeline.is_complete() { @@ -52,46 +65,32 @@ impl SyncM Ok(vec![]) } - /// Keep the entire pipeline (downloaded blocks, pending queue, per-block - /// wallet routing) and the `filters_sync_complete` flag, and move in-flight - /// `getdata`s back to the front of `pending` so the next `send_pending` - /// reissues them to the new peer immediately. Without this preservation, - /// `FiltersManager`'s tracker would re-track the same block hashes after a - /// re-scan and leak `pending_blocks` counters that never reach zero. - fn on_disconnect(&mut self) { - self.pipeline.requeue_in_flight(); - } + /// Keep the entire pipeline (downloaded blocks, pending queue, in-flight + /// requests, per-block wallet routing) and the `filters_sync_complete` flag + /// across a peer disconnect. Without this preservation, `FiltersManager`'s + /// tracker would re-track the same block hashes after a re-scan and leak + /// `pending_blocks` counters that never reach zero. + fn on_disconnect(&mut self) {} async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + _peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult> { - let NetworkMessage::Block(block) = msg.inner() else { + let NetworkMessage::Block(block) = &msg else { return Ok(vec![]); }; let hashed_block = HashedBlock::from(block); - // Check if this is a block we requested (pipeline handles buffering with height) - if !self.pipeline.receive_block(&hashed_block) { + // The pipeline requested this block for a height it recorded at queue time, so it + // hands the height straight back — no need to ask the storage's chain-wide hash + // index to re-derive it. + let Some(height) = self.pipeline.receive_block(&hashed_block) else { tracing::debug!("Received unrequested block {}", hashed_block.hash()); return Ok(vec![]); - } - - // Look up height for storage - let height = self - .header_storage - .read() - .await - .get_header_height_by_hash(hashed_block.hash()) - .await? - .ok_or_else(|| { - SyncError::InvalidState(format!( - "Block {} has no stored header - cannot determine height", - hashed_block.hash() - )) - })?; + }; tracing::debug!("Received block {} at height {}", hashed_block.hash(), height); @@ -104,7 +103,7 @@ impl SyncM let events = self.process_buffered_blocks().await?; if self.pipeline.has_pending_requests() { - self.send_pending(requests).await?; + self.send_pending(network).await?; } Ok(events) @@ -113,7 +112,7 @@ impl SyncM async fn handle_sync_event( &mut self, event: &SyncEvent, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { // React to BlocksNeeded events if let SyncEvent::BlocksNeeded { @@ -163,7 +162,7 @@ impl SyncM // Send batched request for blocks not in storage if self.pipeline.has_pending_requests() { - self.send_pending(requests).await?; + self.send_pending(network).await?; } // Process any blocks we loaded from storage @@ -192,11 +191,10 @@ impl SyncM Ok(vec![]) } - async fn tick(&mut self, requests: &RequestSender) -> SyncResult> { - // Handle timeouts - self.pipeline.handle_timeouts(); - - self.send_pending(requests).await?; + async fn tick(&mut self, network: &Arc) -> SyncResult> { + // Re-issue any timed-out block downloads, then send pending. + self.pipeline.handle_timeouts(std::time::Duration::from_secs(30)); + self.send_pending(network).await?; // Try to process any buffered blocks self.process_buffered_blocks().await diff --git a/dash-spv/src/sync/chainlock/manager.rs b/dash-spv/src/sync/chainlock/manager.rs index c211919df..25b21bc8d 100644 --- a/dash-spv/src/sync/chainlock/manager.rs +++ b/dash-spv/src/sync/chainlock/manager.rs @@ -288,360 +288,3 @@ impl std::fmt::Debug for ChainLockMan .finish() } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::network::MessageType; - use crate::storage::{ - DiskStorageManager, PersistentBlockHeaderStorage, PersistentMetadataStorage, StorageManager, - }; - use crate::sync::{ManagerIdentifier, SyncManager, SyncManagerProgress, SyncState}; - use crate::Network; - use dashcore::bls_sig_utils::BLSSignature; - use dashcore::hashes::Hash; - use dashcore::BlockHash; - - type TestChainLockManager = - ChainLockManager; - - async fn create_test_manager() -> TestChainLockManager { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let engine = - Arc::new(RwLock::new(MasternodeListEngine::default_for_network(Network::Testnet))); - ChainLockManager::new(storage.block_headers(), storage.metadata(), engine).await - } - - async fn create_test_manager_with_storage( - storage: &DiskStorageManager, - ) -> TestChainLockManager { - let engine = - Arc::new(RwLock::new(MasternodeListEngine::default_for_network(Network::Testnet))); - ChainLockManager::new(storage.block_headers(), storage.metadata(), engine).await - } - - fn create_test_chainlock(height: u32) -> ChainLock { - ChainLock { - block_height: height, - block_hash: BlockHash::all_zeros(), - signature: BLSSignature::from([0u8; 96]), - } - } - - #[tokio::test] - async fn test_chainlock_manager_new() { - let manager = create_test_manager().await; - assert_eq!(manager.identifier(), ManagerIdentifier::ChainLock); - assert_eq!(manager.state(), SyncState::WaitForEvents); - assert_eq!(manager.wanted_message_types(), vec![MessageType::CLSig, MessageType::Inv]); - } - - /// Buffered `MasternodeStateUpdated` events delivered during - /// `WaitingForConnections` must not force a `Synced` transition. - /// `MasternodesManager` re-emits the event once it completes its next - /// sync cycle after reconnect, so dropping it here is safe. - #[tokio::test] - async fn test_handle_sync_event_drops_masternode_state_updated_in_waiting_for_connections() { - use crate::network::RequestSender; - use crate::sync::SyncEvent; - use tokio::sync::mpsc::unbounded_channel; - - let mut manager = create_test_manager().await; - manager.set_state(SyncState::WaitingForConnections); - - let event = SyncEvent::MasternodeStateUpdated { - height: 100, - qr_info_result: None, - }; - let (tx, _rx) = unbounded_channel(); - let events = manager.handle_sync_event(&event, &RequestSender::new(tx)).await.unwrap(); - - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - assert!(!manager.masternode_ready); - } - - #[tokio::test] - async fn test_chainlock_skips_validation_before_masternode_ready() { - let mut manager = create_test_manager().await; - - // Before masternode sync, ChainLocks should not be validated - let chainlock = create_test_chainlock(100); - let events = manager.process_chainlock(&chainlock).await.unwrap(); - - assert_eq!(events.len(), 1); - assert_eq!(manager.progress.valid(), 0); - assert_eq!(manager.progress.invalid(), 0); - assert!(manager.best_chainlock().is_none()); - // But the chainlock must be cached for retry once masternode - // state arrives, rather than discarded. - assert!(manager.pending_validation.is_some()); - } - - #[tokio::test] - async fn test_pending_validation_keeps_highest() { - let mut manager = create_test_manager().await; - - // Lower height first, then higher — pending_validation tracks - // the highest seen pre-ready chainlock so the retry on - // masternode-ready always validates the most recent. - let _ = manager.process_chainlock(&create_test_chainlock(100)).await.unwrap(); - let _ = manager.process_chainlock(&create_test_chainlock(200)).await.unwrap(); - let _ = manager.process_chainlock(&create_test_chainlock(150)).await.unwrap(); - - assert_eq!(manager.pending_validation.as_ref().map(|cl| cl.block_height), Some(200)); - } - - #[tokio::test] - async fn test_on_masternode_ready_rejects_pending_chainlock_on_block_hash_mismatch() { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let mut manager = create_test_manager_with_storage(&storage).await; - - // Cache a chainlock for height 100 BEFORE any header exists. - // `process_chainlock`'s permissive `verify_block_hash` lets it - // through and it lands in `pending_validation`. - let _ = manager.process_chainlock(&create_test_chainlock(100)).await.unwrap(); - assert!(manager.pending_validation.is_some()); - - // Header for height 100 resolves later with a hash that differs - // from the cached chainlock's `BlockHash::all_zeros()`. The - // readiness transition must re-check `verify_block_hash` and - // drop the chainlock instead of moving the finality boundary. - let header = dashcore::block::Header::dummy(100); - storage - .block_headers() - .write() - .await - .store_headers_at_height(&[crate::types::HashedBlockHeader::from(header)], 100) - .await - .expect("store header at 100"); - - let rebroadcast = manager.on_masternode_ready().await; - assert!(rebroadcast.is_none(), "mismatched chainlock must not be re-broadcast"); - assert!(manager.best_chainlock().is_none(), "mismatched chainlock must not be persisted"); - assert!(manager.pending_validation.is_none(), "pending_validation must be consumed"); - assert_eq!(manager.progress.invalid(), 1); - assert_eq!(manager.progress.valid(), 0); - } - - #[tokio::test] - async fn test_on_masternode_ready_retries_pending_validation() { - let mut manager = create_test_manager().await; - let _ = manager.process_chainlock(&create_test_chainlock(100)).await.unwrap(); - assert!(manager.pending_validation.is_some()); - - // With the default empty engine, validation fails — the - // pending chainlock is consumed (cleared) and counted as - // invalid; `best_chainlock` stays `None`. - let rebroadcast = manager.on_masternode_ready().await; - assert!(rebroadcast.is_none()); - assert!(manager.pending_validation.is_none()); - assert!(manager.best_chainlock().is_none()); - assert_eq!(manager.progress.invalid(), 1); - assert!(manager.masternode_ready); - } - - #[tokio::test] - async fn test_chainlock_validates_after_masternode_ready() { - let mut manager = create_test_manager().await; - let _ = manager.on_masternode_ready().await; - - // After masternode sync, ChainLocks should be validated (will fail with empty engine) - let chainlock = create_test_chainlock(100); - let _ = manager.process_chainlock(&chainlock).await.unwrap(); - - assert_eq!(manager.progress.invalid(), 1); - assert_eq!(manager.progress.valid(), 0); - } - - #[tokio::test] - async fn test_chainlock_keeps_only_best() { - let mut manager = create_test_manager().await; - - // Manually set a best chainlock - manager.best_chainlock = Some(create_test_chainlock(200)); - - // Lower height should be ignored - let chainlock_lower = create_test_chainlock(150); - let events = manager.process_chainlock(&chainlock_lower).await.unwrap(); - assert_eq!(events.len(), 0); - - // Equal height should also be ignored - let chainlock_equal = create_test_chainlock(200); - let events = manager.process_chainlock(&chainlock_equal).await.unwrap(); - assert_eq!(events.len(), 0); - - // Higher height should be processed - let chainlock_higher = create_test_chainlock(300); - let events = manager.process_chainlock(&chainlock_higher).await.unwrap(); - assert_eq!(events.len(), 1); - } - - #[tokio::test] - async fn test_chainlock_progress() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - manager.progress.update_best_validated_height(500); - manager.progress.add_valid(8); - manager.progress.add_invalid(2); - - let progress = manager.progress(); - if let SyncManagerProgress::ChainLock(cp) = progress { - assert_eq!(cp.state(), SyncState::Syncing); - assert_eq!(cp.best_validated_height(), 500); - assert_eq!(cp.valid(), 8); - assert_eq!(cp.invalid(), 2); - } else { - panic!("Expected SyncManagerProgress::ChainLock"); - } - } - - #[tokio::test] - async fn test_is_block_chainlocked() { - let mut manager = create_test_manager().await; - - // No ChainLock yet - assert!(!manager.is_block_chainlocked(100)); - - // Manually set best chainlock for testing - manager.best_chainlock = Some(create_test_chainlock(500)); - - // All blocks at or below 500 should be chainlocked - assert!(manager.is_block_chainlocked(1)); - assert!(manager.is_block_chainlocked(500)); - assert!(!manager.is_block_chainlocked(501)); - } - - #[tokio::test] - async fn test_load_from_empty_storage_returns_none() { - let mut manager = create_test_manager().await; - - manager.load_best_chainlock().await; - - assert!(manager.best_chainlock().is_none()); - assert_eq!(manager.progress.best_validated_height(), 0); - } - - #[tokio::test] - async fn test_save_and_load_chainlock_round_trip() { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let chainlock = create_test_chainlock(42000); - - // Save a chainlock via the first manager - { - let mut manager = create_test_manager_with_storage(&storage).await; - manager.best_chainlock = Some(chainlock.clone()); - manager.save_best_chainlock().await; - } - - // Fresh manager sharing the same storage should load the chainlock automatically - { - let manager = create_test_manager_with_storage(&storage).await; - - let restored = manager.best_chainlock().expect("chainlock should be restored"); - assert_eq!(restored.block_height, 42000); - assert_eq!(restored.block_hash, chainlock.block_hash); - assert_eq!(restored.signature, chainlock.signature); - assert_eq!(manager.progress.best_validated_height(), 42000); - } - } - - #[tokio::test] - async fn test_initialize_restores_persisted_chainlock() { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let chainlock = create_test_chainlock(99999); - - // Persist a chainlock directly via metadata storage - { - let bytes = serde_json::to_vec(&chainlock).unwrap(); - let meta_storage = storage.metadata(); - let mut meta = meta_storage.write().await; - meta.store_metadata(BEST_CHAINLOCK_KEY, &bytes).await.unwrap(); - } - - // Create a new manager and call initialize (the SyncManager trait method) - let manager = create_test_manager_with_storage(&storage).await; - - let restored = - manager.best_chainlock().expect("chainlock should be restored after initialize"); - assert_eq!(restored.block_height, 99999); - assert_eq!(manager.progress.best_validated_height(), 99999); - assert_eq!(manager.state(), SyncState::WaitForEvents); - } - - #[tokio::test] - async fn test_process_chainlock_persists_on_validation() { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let mut manager = create_test_manager_with_storage(&storage).await; - - // Without masternode ready, chainlocks are not validated and not persisted - let chainlock = create_test_chainlock(500); - manager.process_chainlock(&chainlock).await.unwrap(); - assert!(manager.best_chainlock().is_none()); - - // Verify nothing was persisted - { - let meta_storage = storage.metadata(); - let meta = meta_storage.read().await; - let loaded = meta.load_metadata(BEST_CHAINLOCK_KEY).await.unwrap(); - assert!(loaded.is_none()); - } - } - - #[tokio::test] - async fn test_save_overwrites_previous_chainlock() { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - - // Save first chainlock - { - let mut manager = create_test_manager_with_storage(&storage).await; - manager.best_chainlock = Some(create_test_chainlock(100)); - manager.save_best_chainlock().await; - } - - // Save a higher chainlock - { - let mut manager = create_test_manager_with_storage(&storage).await; - manager.best_chainlock = Some(create_test_chainlock(200)); - manager.save_best_chainlock().await; - } - - // Load and verify only the latest is stored - { - let mut manager = create_test_manager_with_storage(&storage).await; - manager.load_best_chainlock().await; - - let restored = manager.best_chainlock().expect("chainlock should be restored"); - assert_eq!(restored.block_height, 200); - } - } - - #[tokio::test] - async fn test_lower_chainlock_rejected_after_load() { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - - // Save chainlock at height 200 - { - let mut manager = create_test_manager_with_storage(&storage).await; - manager.best_chainlock = Some(create_test_chainlock(200)); - manager.save_best_chainlock().await; - } - - // Load and try to process a lower chainlock - { - let mut manager = create_test_manager_with_storage(&storage).await; - manager.load_best_chainlock().await; - - // Try to process a lower chainlock - let lower_chainlock = create_test_chainlock(100); - let events = manager.process_chainlock(&lower_chainlock).await.unwrap(); - - // Should be rejected (no events) - assert_eq!(events.len(), 0); - - // Best should still be 200 - let best = manager.best_chainlock().expect("should have best chainlock"); - assert_eq!(best.block_height, 200); - } - } -} diff --git a/dash-spv/src/sync/chainlock/sync_manager.rs b/dash-spv/src/sync/chainlock/sync_manager.rs index b750410a5..c9bd94847 100644 --- a/dash-spv/src/sync/chainlock/sync_manager.rs +++ b/dash-spv/src/sync/chainlock/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network::{Message, MessageType, RequestSender}; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, MetadataStorage}; use crate::sync::{ ChainLockManager, ManagerIdentifier, SyncEvent, SyncManager, SyncManagerProgress, SyncState, @@ -7,6 +7,8 @@ use crate::sync::{ use async_trait::async_trait; use dashcore::network::message::NetworkMessage; use dashcore::network::message_blockdata::Inventory; +use std::net::SocketAddr; +use std::sync::Arc; #[async_trait] impl SyncManager for ChainLockManager { @@ -22,8 +24,9 @@ impl SyncManager for ChainLockManager self.progress.set_state(state); } - fn wanted_message_types(&self) -> &'static [MessageType] { - &[MessageType::CLSig, MessageType::Inv] + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { + use crate::network2::MessageType; + &[MessageType::ChainLock, MessageType::Inv] } fn on_disconnect(&mut self) { @@ -33,10 +36,11 @@ impl SyncManager for ChainLockManager async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult> { - match msg.inner() { + match &msg { NetworkMessage::CLSig(chainlock) => self.process_chainlock(chainlock).await, NetworkMessage::Inv(inv) => { // Check for ChainLock inventory items, filtering out already-requested ones @@ -58,8 +62,7 @@ impl SyncManager for ChainLockManager "Received {} ChainLock announcements, requesting via getdata", chainlocks_to_request.len() ); - requests - .request_inventory(chainlocks_to_request.clone(), msg.peer_address())?; + network.send(NetworkMessage::GetData(chainlocks_to_request.clone())).await; for item in &chainlocks_to_request { if let Inventory::ChainLock(hash) = item { @@ -76,7 +79,7 @@ impl SyncManager for ChainLockManager async fn handle_sync_event( &mut self, event: &SyncEvent, - _requests: &RequestSender, + _network: &Arc, ) -> SyncResult> { // `MasternodeStateUpdated` fires on every MnListDiff / QRInfo // update; the work below is strictly one-shot startup work, so @@ -112,7 +115,7 @@ impl SyncManager for ChainLockManager Ok(vec![]) } - async fn tick(&mut self, _requests: &RequestSender) -> SyncResult> { + async fn tick(&mut self, _network: &Arc) -> SyncResult> { // No periodic work needed Ok(vec![]) } diff --git a/dash-spv/src/sync/download_coordinator.rs b/dash-spv/src/sync/download_coordinator.rs index e36753b6d..69df27af4 100644 --- a/dash-spv/src/sync/download_coordinator.rs +++ b/dash-spv/src/sync/download_coordinator.rs @@ -1,54 +1,17 @@ //! Generic download coordinator for pipelined downloads. //! -//! Provides a single abstraction for managing concurrent downloads with: -//! - Pending queue management -//! - In-flight tracking with timestamps -//! - Timeout detection and retry logic -//! - Configurable concurrency limits +//! A pending queue plus an in-flight map (item -> send time). Pacing is the +//! network manager's job, but retry lives here: [`take_timed_out`] surfaces +//! requests that were sent but never answered (a peer died, or the response was +//! lost) so the pipeline can re-issue them. +//! +//! [`take_timed_out`]: DownloadCoordinator::take_timed_out use std::collections::{HashMap, VecDeque}; use std::hash::Hash; use std::time::{Duration, Instant}; -/// Configuration for download coordination. -#[derive(Debug, Clone)] -pub struct DownloadConfig { - /// Maximum concurrent in-flight requests. - max_concurrent: usize, - /// Timeout duration for requests. - timeout: Duration, -} - -impl Default for DownloadConfig { - fn default() -> Self { - Self { - max_concurrent: 10, - timeout: Duration::from_secs(30), - } - } -} - -impl DownloadConfig { - /// Create config with custom max concurrent. - pub(crate) fn with_max_concurrent(mut self, max: usize) -> Self { - self.max_concurrent = max; - self - } - - /// Create config with custom timeout. - pub(crate) fn with_timeout(mut self, timeout: Duration) -> Self { - self.timeout = timeout; - self - } -} - -/// Generic download coordinator. -/// -/// Handles the common mechanics of pipelined downloads: -/// - Queue management (pending items) -/// - In-flight tracking with timestamps -/// - Timeout detection and retry -/// - Concurrency limits +/// Generic download coordinator: a pending queue plus an in-flight map. /// /// Generic over the key type `K` which identifies download items. /// Use `u32` for height-based downloads, `BlockHash` for hash-based. @@ -56,57 +19,34 @@ impl DownloadConfig { pub(crate) struct DownloadCoordinator { /// Items waiting to be requested. pending: VecDeque, - /// Items currently in-flight (key -> sent time). - in_flight: HashMap, - /// Retry counts per key. - retry_counts: HashMap, - /// Configuration. - config: DownloadConfig, - /// Last time progress was made. - last_progress: Instant, + /// Items handed to the network, mapped to their on-the-wire time. `None` = + /// still queued in the router (not yet sent); `Some(t)` = actually sent to a + /// peer at `t` (set from the network manager's `RequestOnFlight`). The + /// response timeout counts from `Some(t)`; queued items never time out, since + /// the queue is drained in order and their send — hence `RequestOnFlight` — + /// is guaranteed to come. + in_flight: HashMap>, } impl Default for DownloadCoordinator { fn default() -> Self { - Self::new(DownloadConfig::default()) - } -} - -impl DownloadCoordinator { - /// Create a new coordinator with the given configuration. - pub(crate) fn new(config: DownloadConfig) -> Self { Self { pending: VecDeque::new(), in_flight: HashMap::new(), - retry_counts: HashMap::new(), - config, - last_progress: Instant::now(), } } +} + +impl DownloadCoordinator { + /// Create an empty coordinator. + pub(crate) fn new() -> Self { + Self::default() + } /// Clear all state. pub(crate) fn clear(&mut self) { self.pending.clear(); self.in_flight.clear(); - self.retry_counts.clear(); - self.last_progress = Instant::now(); - } - - /// Move all in-flight items back to the front of the pending queue. - /// - /// Used on peer disconnect: the requests went to a now-dead peer, but the - /// items themselves are still wanted. Retry counts are preserved so a peer - /// that consistently fails to deliver an item still trips the normal retry - /// budget. Without this hook, items would only be retried once their - /// timeout elapsed. - pub(crate) fn requeue_in_flight(&mut self) { - let items: Vec = self.in_flight.drain().map(|(k, _)| k).collect(); - if items.is_empty() { - return; - } - for item in items.into_iter().rev() { - self.pending.push_front(item); - } } /// Queue items for download. @@ -116,17 +56,12 @@ impl DownloadCoordinator { } } - /// Queue an item for retry (goes to front of queue). - pub(crate) fn enqueue_retry(&mut self, item: K) { - let count = self.retry_counts.entry(item.clone()).or_insert(0); - *count += 1; - tracing::warn!("Retrying item (attempt {})", count); - self.pending.push_front(item); - } - - /// Get the number of items available to send (respecting concurrency limit). + /// Get the number of items available to send. + /// + /// No concurrency cap: the network manager coordinates pacing, so + /// everything pending is offered. pub(crate) fn available_to_send(&self) -> usize { - self.config.max_concurrent.saturating_sub(self.in_flight.len()).min(self.pending.len()) + self.pending.len() } /// Take items from the pending queue (up to count). @@ -144,11 +79,24 @@ impl DownloadCoordinator { items } - /// Mark items as sent (now in-flight). + /// Mark items as handed to the network (queued). The response timeout does + /// NOT start yet — it starts when `mark_on_flight` reports the request was + /// actually sent to a peer. pub(crate) fn mark_sent(&mut self, items: &[K]) { + for item in items { + self.in_flight.insert(item.clone(), None); + } + } + + /// Mark items as actually ON THE WIRE (sent to a peer), starting their + /// response timeout. Driven by the network manager's `RequestOnFlight` event. + /// No-op for items already answered/removed or already on-flight. + pub(crate) fn mark_on_flight(&mut self, items: &[K]) { let now = Instant::now(); for item in items { - self.in_flight.insert(item.clone(), now); + if let Some(slot @ None) = self.in_flight.get_mut(item) { + *slot = Some(now); + } } } @@ -156,66 +104,47 @@ impl DownloadCoordinator { /// /// Returns true if the item was being tracked, false if unexpected. pub(crate) fn receive(&mut self, key: &K) -> bool { - if self.in_flight.remove(key).is_some() { - self.retry_counts.remove(key); - self.last_progress = Instant::now(); - true - } else { - false - } - } - - /// Drop a key from the pending queue without touching in-flight state. - /// - /// Used when a pending item is satisfied through a side channel: a late - /// response from a disconnected peer can complete a batch that - /// `requeue_in_flight` just moved from in-flight back to pending. Without - /// this hook, the key would stay in `pending` with no tracker, and the - /// next `take_pending` would resurrect a finished batch. - pub(crate) fn cancel_pending(&mut self, key: &K) { - self.pending.retain(|k| k != key); - self.retry_counts.remove(key); + self.in_flight.remove(key).is_some() } - /// Check if an item is currently in-flight. - pub(crate) fn is_in_flight(&self, key: &K) -> bool { - self.in_flight.contains_key(key) - } - - /// Check for timed-out items. - /// - /// Returns items that have timed out. They are removed from in-flight tracking. - /// Caller should call `enqueue_retry` for items that should be retried. - pub(crate) fn check_timeouts(&mut self) -> Vec { + /// Remove and return every in-flight item that has been outstanding longer + /// than `timeout` (a peer died or the response was lost). The items leave the + /// in-flight set, so the pipeline simply re-sends them next tick — segments + /// via `can_send`, pending-driven pipelines by re-`enqueue`ing what's returned. + pub(crate) fn take_timed_out(&mut self, timeout: Duration) -> Vec { let now = Instant::now(); + // Only requests that are actually ON THE WIRE can time out. A request + // still sitting in the router queue (`on_flight` = None) is NEVER timed + // out here: the queue drains in order and items are not dropped, so its + // `RequestOnFlight` is guaranteed to fire eventually — timing it out from + // queue time would re-issue a request that is merely waiting its turn + // behind a deep queue, a spurious duplicate. let timed_out: Vec = self .in_flight .iter() - .filter(|(_, sent_time)| now.duration_since(**sent_time) > self.config.timeout) - .map(|(key, _)| key.clone()) + .filter(|(_, on_flight)| { + on_flight.is_some_and(|sent| now.duration_since(sent) > timeout) + }) + .map(|(k, _)| k.clone()) .collect(); - - for key in &timed_out { - self.in_flight.remove(key); - } - - if !timed_out.is_empty() { - tracing::debug!("{} items timed out after {:?}", timed_out.len(), self.config.timeout); + for k in &timed_out { + self.in_flight.remove(k); } - timed_out } - /// Check for timed-out items and re-enqueue them for retry. + /// Drop a key from the pending queue without touching in-flight state. /// - /// Combines `check_timeouts()` and `enqueue_retry()` in one call. - /// Returns all timed-out items that were re-queued. - pub(crate) fn check_and_retry_timeouts(&mut self) -> Vec { - let timed_out = self.check_timeouts(); - for item in &timed_out { - self.enqueue_retry(item.clone()); - } - timed_out + /// Used when a pending item is satisfied through a side channel (e.g. a + /// batch completed via a different path) so the next `take_pending` doesn't + /// resurrect a finished item. + pub(crate) fn cancel_pending(&mut self, key: &K) { + self.pending.retain(|k| k != key); + } + + /// Check if an item is currently in-flight. + pub(crate) fn is_in_flight(&self, key: &K) -> bool { + self.in_flight.contains_key(key) } /// Check if the coordinator has no work (empty pending and in-flight). @@ -245,33 +174,24 @@ mod tests { #[test] fn test_new_coordinator() { - let coord: DownloadCoordinator = DownloadCoordinator::default(); + let coord: DownloadCoordinator = DownloadCoordinator::new(); assert!(coord.is_empty()); assert_eq!(coord.pending_count(), 0); assert_eq!(coord.active_count(), 0); } #[test] - fn test_enqueue() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); + fn test_enqueue_and_available() { + let mut coord: DownloadCoordinator = DownloadCoordinator::new(); coord.enqueue([1, 2, 3, 4, 5]); - assert_eq!(coord.pending_count(), 5); - } - - #[test] - fn test_enqueue_retry_goes_to_front() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); - coord.enqueue([1, 2]); - coord.enqueue_retry(99); - - let items = coord.take_pending(3); - assert_eq!(items, vec![99, 1, 2]); + // No concurrency cap: everything pending is available. + assert_eq!(coord.available_to_send(), 5); } #[test] fn test_take_pending() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); + let mut coord: DownloadCoordinator = DownloadCoordinator::new(); coord.enqueue([1, 2, 3, 4, 5]); let items = coord.take_pending(3); @@ -280,186 +200,40 @@ mod tests { } #[test] - fn test_mark_sent() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); + fn test_mark_sent_and_receive() { + let mut coord: DownloadCoordinator = DownloadCoordinator::new(); coord.enqueue([1, 2, 3]); let items = coord.take_pending(2); coord.mark_sent(&items); - - assert_eq!(coord.pending_count(), 1); assert_eq!(coord.active_count(), 2); assert!(coord.is_in_flight(&1)); assert!(coord.is_in_flight(&2)); assert!(!coord.is_in_flight(&3)); - } - - #[test] - fn test_receive() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); - coord.mark_sent(&[1]); - coord.mark_sent(&[2]); assert!(coord.receive(&1)); - assert_eq!(coord.active_count(), 1); - - assert!(!coord.receive(&99)); // Not tracked - assert_eq!(coord.active_count(), 1); - } - - #[test] - fn test_available_to_send() { - let mut coord: DownloadCoordinator = - DownloadCoordinator::new(DownloadConfig::default().with_max_concurrent(3)); - - coord.enqueue([1, 2, 3, 4, 5]); - assert_eq!(coord.available_to_send(), 3); - - coord.mark_sent(&[1]); - coord.mark_sent(&[2]); - assert_eq!(coord.available_to_send(), 1); - - coord.mark_sent(&[3]); - assert_eq!(coord.available_to_send(), 0); + assert!(!coord.is_in_flight(&1)); + // Receiving an untracked key returns false. + assert!(!coord.receive(&99)); } #[test] - fn test_check_timeouts() { - let mut coord: DownloadCoordinator = DownloadCoordinator::new( - DownloadConfig::default().with_timeout(Duration::from_millis(10)), - ); - - coord.mark_sent(&[1]); - coord.mark_sent(&[2]); - - // Immediately, nothing timed out - let timed_out = coord.check_timeouts(); - assert!(timed_out.is_empty()); - - // Wait for timeout - std::thread::sleep(Duration::from_millis(20)); - - let timed_out = coord.check_timeouts(); - assert_eq!(timed_out.len(), 2); - assert!(coord.in_flight.is_empty()); - } - - #[test] - fn test_requeue_in_flight_moves_items_to_pending_front() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); - coord.enqueue([10, 11]); - coord.mark_sent(&[1, 2, 3]); - - coord.requeue_in_flight(); - - assert_eq!(coord.active_count(), 0); - // Requeued items go to the front, original pending follows. - let items = coord.take_pending(5); - assert_eq!(items.len(), 5); - assert_eq!(&items[3..], &[10, 11]); - let mut requeued = items[..3].to_vec(); - requeued.sort(); - assert_eq!(requeued, vec![1, 2, 3]); - } - - #[test] - fn test_requeue_in_flight_preserves_retry_counts() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); - coord.enqueue_retry(7); - let items = coord.take_pending(1); - coord.mark_sent(&items); - assert_eq!(coord.retry_counts.get(&7), Some(&1)); - - coord.requeue_in_flight(); - - assert_eq!(coord.retry_counts.get(&7), Some(&1)); - assert!(!coord.is_in_flight(&7)); - assert_eq!(coord.pending_count(), 1); - } - - #[test] - fn test_requeue_in_flight_no_op_when_empty() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); - coord.enqueue([1, 2]); - - coord.requeue_in_flight(); - - assert_eq!(coord.pending_count(), 2); - assert_eq!(coord.active_count(), 0); - } - - #[test] - fn test_cancel_pending_removes_from_pending_only() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); + fn test_cancel_pending() { + let mut coord: DownloadCoordinator = DownloadCoordinator::new(); coord.enqueue([1, 2, 3]); - coord.mark_sent(&[10]); - coord.enqueue_retry(2); - assert_eq!(coord.retry_counts.get(&2), Some(&1)); - coord.cancel_pending(&2); - - assert_eq!(coord.pending_count(), 2); - assert!(coord.is_in_flight(&10)); - assert_eq!(coord.retry_counts.get(&2), None); - - let items = coord.take_pending(2); - assert_eq!(items, vec![1, 3]); + assert_eq!(coord.take_pending(3), vec![1, 3]); } #[test] - fn test_cancel_pending_unknown_key_is_noop() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); - coord.enqueue([1, 2]); - coord.mark_sent(&[5]); - - coord.cancel_pending(&99); - - assert_eq!(coord.pending_count(), 2); - assert_eq!(coord.active_count(), 1); - } - - #[test] - fn test_clear() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); + fn test_clear_and_remaining() { + let mut coord: DownloadCoordinator = DownloadCoordinator::new(); coord.enqueue([1, 2, 3]); coord.mark_sent(&[4]); - coord.enqueue_retry(5); + assert_eq!(coord.remaining(), 4); coord.clear(); - assert!(coord.is_empty()); - assert_eq!(coord.pending_count(), 0); - assert_eq!(coord.active_count(), 0); - } - - #[test] - fn test_remaining() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); - coord.enqueue([1, 2, 3]); - coord.mark_sent(&[4]); - coord.mark_sent(&[5]); - - assert_eq!(coord.remaining(), 5); - } - - #[test] - fn test_config_builders() { - let config = - DownloadConfig::default().with_max_concurrent(20).with_timeout(Duration::from_secs(60)); - - assert_eq!(config.max_concurrent, 20); - assert_eq!(config.timeout, Duration::from_secs(60)); - } - - #[test] - fn test_with_string_keys() { - let mut coord: DownloadCoordinator = DownloadCoordinator::default(); - coord.enqueue(["block_a".to_string(), "block_b".to_string()]); - - let items = coord.take_pending(1); - coord.mark_sent(&items); - - assert!(coord.receive(&"block_a".to_string())); - assert!(!coord.receive(&"block_c".to_string())); + assert_eq!(coord.remaining(), 0); } } diff --git a/dash-spv/src/sync/events.rs b/dash-spv/src/sync/events.rs index 2fbf24217..6d627edf7 100644 --- a/dash-spv/src/sync/events.rs +++ b/dash-spv/src/sync/events.rs @@ -1,11 +1,13 @@ use crate::sync::ManagerIdentifier; use dashcore::ephemerealdata::chain_lock::ChainLock; use dashcore::ephemerealdata::instant_lock::InstantLock; +use dashcore::hash_types::FilterHeader; use dashcore::sml::masternode_list_engine::QRInfoFeedResult; use dashcore::{BlockHash, ScriptBuf, Txid}; use key_wallet_manager::{FilterMatchKey, WalletId}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; +use std::sync::Arc; /// Events that managers can emit and subscribe to. /// @@ -29,6 +31,22 @@ pub enum SyncEvent { tip_height: u32, }, + /// A batch of block headers has arrived and been validated IN MEMORY, before + /// the (sequential) store. Carries the batch range and the block hash at its + /// end so filter-header sync can request `getcfheaders` for that range in + /// parallel across segments, without waiting for the in-order store. + /// + /// Emitted by: `BlockHeadersManager` + /// Consumed by: `FilterHeadersManager` + BlockHeadersInMemory { + /// First height in the batch. + start_height: u32, + /// Last height in the batch. + end_height: u32, + /// Block hash at `end_height` (the `getcfheaders` stop hash). + stop_hash: BlockHash, + }, + /// Headers have reached the chain tip (initial sync complete). /// /// Emitted by: `BlockHeadersManager` @@ -49,6 +67,13 @@ pub enum SyncEvent { end_height: u32, /// New tip height after storage tip_height: u32, + /// The verified filter headers themselves, `start_height..=end_height`. + /// + /// Carried on the event so `FiltersManager` can verify arriving filters + /// straight from memory: it is the only consumer, it needs exactly this + /// range, and it would otherwise read the same bytes back from storage. An + /// `Arc` so the (potentially large) batch is not cloned per subscriber. + headers: Arc>, }, /// Filter headers have reached the chain tip (initial sync complete). @@ -187,6 +212,11 @@ impl fmt::Display for SyncEvent { SyncEvent::BlockHeadersStored { tip_height, } => write!(f, "BlockHeadersStored(tip={})", tip_height), + SyncEvent::BlockHeadersInMemory { + start_height, + end_height, + .. + } => write!(f, "BlockHeadersInMemory({}-{})", start_height, end_height), SyncEvent::BlockHeaderSyncComplete { tip_height, } => write!(f, "BlockHeaderSyncComplete(tip={})", tip_height), @@ -194,6 +224,7 @@ impl fmt::Display for SyncEvent { start_height, end_height, tip_height, + .. } => write!( f, "FilterHeadersStored({}-{}, tip={})", diff --git a/dash-spv/src/sync/filter_headers/manager.rs b/dash-spv/src/sync/filter_headers/manager.rs index eb4f3600f..fe6933075 100644 --- a/dash-spv/src/sync/filter_headers/manager.rs +++ b/dash-spv/src/sync/filter_headers/manager.rs @@ -5,12 +5,13 @@ use std::sync::Arc; +use dashcore::hash_types::FilterHeader; use dashcore::network::message_filter::CFHeaders; use tokio::sync::RwLock; use super::pipeline::FilterHeadersPipeline; use crate::error::SyncResult; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage}; use crate::sync::filter_headers::util::compute_filter_headers; use crate::sync::progress::ProgressPercentage; @@ -30,7 +31,7 @@ pub struct FilterHeadersManager /// Current progress of the manager. pub(super) progress: FilterHeadersProgress, /// Block header storage (for reading headers). - header_storage: Arc>, + pub(super) header_storage: Arc>, /// Filter header storage (for storing filter headers). pub(super) filter_header_storage: Arc>, /// Pipeline for downloading filter headers. @@ -40,6 +41,13 @@ pub struct FilterHeadersManager /// Whether block header sync has completed. Gates FilterHeadersSyncComplete emission /// to ensure it never fires before BlockHeaderSyncComplete. pub(super) block_headers_synced: bool, + /// Item #3: when set, cfheaders batches are driven by `BlockHeadersInMemory` + /// events (one `getcfheaders` per block-header batch, fanned out across + /// segments) instead of the storage-gated `init`/`extend_target` path. + /// Enabled only on a fresh sync (`filter_headers_tip == 0`); resumes fall + /// back to the storage path. `None` until the mode is decided on the first + /// block-header event. + pub(super) in_memory_driven: Option, } impl FilterHeadersManager { @@ -90,16 +98,21 @@ impl FilterHeadersManager pipeline: FilterHeadersPipeline::default(), checkpoint_start_height: None, block_headers_synced: false, + in_memory_driven: None, }) } /// Process a CFHeaders response - store headers and update state. + /// Verify and store one in-order batch of filter headers, returning them so + /// the caller can hand them straight to `FiltersManager` on the + /// `FilterHeadersStored` event — it needs exactly these to authenticate the + /// filters it is downloading, and would otherwise read them back from storage. pub(super) async fn process_cfheaders( &mut self, cfheaders: &CFHeaders, start_height: u32, - ) -> SyncResult { - let filter_headers = compute_filter_headers(cfheaders); + ) -> SyncResult>> { + let filter_headers = Arc::new(compute_filter_headers(cfheaders)); let count = filter_headers.len() as u32; let mut storage = self.filter_header_storage.write().await; @@ -129,11 +142,14 @@ impl FilterHeadersManager self.progress.add_processed(count); - Ok(count) + Ok(filter_headers) } /// Start or resume filter header download. - async fn start_download(&mut self, requests: &RequestSender) -> SyncResult> { + async fn start_download( + &mut self, + network: &Arc, + ) -> SyncResult> { // Get current filter tip let filter_headers_tip = self.filter_header_storage.read().await.get_filter_tip_height().await?.unwrap_or(0); @@ -179,7 +195,7 @@ impl FilterHeadersManager drop(header_storage); // Send initial requests - self.pipeline.send_pending(requests)?; + self.pipeline.send_pending(network).await?; self.set_state(SyncState::Syncing); @@ -193,7 +209,7 @@ impl FilterHeadersManager pub(super) async fn handle_new_headers( &mut self, tip_height: u32, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { self.progress.update_block_header_tip_height(tip_height); self.update_target_height(tip_height); @@ -227,170 +243,91 @@ impl FilterHeadersManager .await?; } drop(header_storage); - self.pipeline.send_pending(requests)?; + self.pipeline.send_pending(network).await?; Ok(vec![]) } SyncState::WaitingForConnections | SyncState::WaitForEvents => { // Need full startup (calculates start from storage, handles checkpoints) - self.start_download(requests).await + self.start_download(network).await } _ => Ok(vec![]), } } -} -impl std::fmt::Debug - for FilterHeadersManager -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("FilterHeadersManager").field("progress", &self.progress).finish() - } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::network::MessageType; - use crate::storage::{ - DiskStorageManager, PersistentBlockHeaderStorage, PersistentFilterHeaderStorage, - StorageManager, - }; - use crate::sync::{ManagerIdentifier, SyncManagerProgress}; - - type TestFilterHeadersManager = - FilterHeadersManager; - type TestSyncManager = dyn SyncManager; - - async fn create_test_manager() -> TestFilterHeadersManager { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - FilterHeadersManager::new(storage.block_headers(), storage.filter_headers()) - .await - .expect("Failed to create FilterHeadersManager") - } - - fn create_test_request_sender( - ) -> (RequestSender, tokio::sync::mpsc::UnboundedReceiver) { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - (RequestSender::new(tx), rx) - } - - #[tokio::test] - async fn test_filter_headers_manager_new() { - let manager = create_test_manager().await; - assert_eq!(manager.identifier(), ManagerIdentifier::FilterHeader); - assert_eq!(manager.state(), SyncState::WaitForEvents); - assert_eq!(manager.wanted_message_types(), vec![MessageType::CFHeaders]); - assert!(!manager.block_headers_synced); - } - - #[tokio::test] - async fn test_filter_headers_manager_progress() { - let mut manager = create_test_manager().await; - manager.progress.update_current_height(500); - manager.progress.update_target_height(2000); - manager.progress.update_block_header_tip_height(1000); - manager.progress.add_processed(500); - - let manager_ref: &TestSyncManager = &manager; - let progress = manager_ref.progress(); - if let SyncManagerProgress::FilterHeaders(progress) = progress { - assert_eq!(progress.state(), SyncState::WaitForEvents); - assert_eq!(progress.current_height(), 500); - assert_eq!(progress.target_height(), 2000); - assert_eq!(progress.block_header_tip_height(), 1000); - assert_eq!(progress.processed(), 500); - assert!(progress.last_activity().elapsed().as_secs() < 1); - } else { - panic!("Expected SyncManagerProgress::FilterHeaders"); + /// Decide, on the first block-header event, whether to drive cfheaders from + /// in-memory block-header arrivals (fresh sync) or from the storage-gated + /// path (resume). Idempotent — the choice is cached in `in_memory_driven`. + pub(super) async fn ensure_mode(&mut self) -> SyncResult<()> { + if self.in_memory_driven.is_some() { + return Ok(()); } + let filter_tip = + self.filter_header_storage.read().await.get_filter_tip_height().await?.unwrap_or(0); + if filter_tip != 0 { + // Resume: block headers are (usually) already stored, so no in-memory + // events fire; the storage path handles it. Keep behaviour unchanged. + self.in_memory_driven = Some(false); + return Ok(()); + } + let header_start = self.header_storage.read().await.get_start_height().await.unwrap_or(0); + // Block-header in-memory batches begin at `header_start + 1`; anchor the + // filter chain's `previous_filter_header` at `header_start`. + let start = header_start + 1; + self.pipeline.init_in_memory(start, self.progress.block_header_tip_height()); + self.checkpoint_start_height = Some(start); + self.set_state(SyncState::Syncing); + self.in_memory_driven = Some(true); + tracing::info!( + "Filter headers: in-memory-driven sync from height {} (parallel per-segment cfheaders)", + start + ); + Ok(()) } - #[tokio::test] - async fn test_try_complete_sync() { - let mut manager = create_test_manager().await; - manager.progress.update_current_height(1000); - manager.progress.update_target_height(1000); - manager.progress.update_block_header_tip_height(1000); - manager.set_state(SyncState::Syncing); - - // Gated: returns None when block_headers_synced is false - assert!(manager.try_complete_sync().is_none()); - assert_eq!(manager.state(), SyncState::Syncing); - - // Emits once block_headers_synced is set - manager.block_headers_synced = true; - assert!(matches!( - manager.try_complete_sync(), - Some(SyncEvent::FilterHeadersSyncComplete { .. }) - )); - assert_eq!(manager.state(), SyncState::Synced); - - // Idempotent: returns None when already Synced - assert!(manager.try_complete_sync().is_none()); - assert_eq!(manager.state(), SyncState::Synced); - } - - #[tokio::test] - async fn test_block_headers_synced_event_gating() { - let mut manager = create_test_manager().await; - let (sender, _rx) = create_test_request_sender(); - - // Filter headers caught up to block header tip and target - manager.progress.update_current_height(1000); - manager.progress.update_target_height(1000); - manager.progress.update_block_header_tip_height(1000); - manager.set_state(SyncState::WaitForEvents); - - // BlockHeadersStored does NOT set block_headers_synced, no completion emitted - let event = SyncEvent::BlockHeadersStored { - tip_height: 1000, - }; - let events = manager.handle_sync_event(&event, &sender).await.unwrap(); - assert!(!manager.block_headers_synced); - assert!(!events.iter().any(|e| matches!(e, SyncEvent::FilterHeadersSyncComplete { .. }))); - - // BlockHeaderSyncComplete sets the flag and emits completion - let event = SyncEvent::BlockHeaderSyncComplete { - tip_height: 1000, - }; - let events = manager.handle_sync_event(&event, &sender).await.unwrap(); - assert!(manager.block_headers_synced); - assert!(events.iter().any(|e| matches!(e, SyncEvent::FilterHeadersSyncComplete { .. }))); - assert_eq!(manager.state(), SyncState::Synced); + /// Item #3: queue cfheaders for a block-header batch that just landed in + /// memory, without waiting for the sequential block-header store. One + /// `getcfheaders` per block-header batch, fanned out across segments. + pub(super) async fn handle_headers_in_memory( + &mut self, + start_height: u32, + end_height: u32, + stop_hash: dashcore::BlockHash, + network: &Arc, + ) -> SyncResult> { + self.ensure_mode().await?; + if self.in_memory_driven != Some(true) { + return Ok(vec![]); + } + self.pipeline.queue_batch(start_height, end_height, stop_hash); + self.pipeline.send_pending(network).await?; + Ok(vec![]) } - #[tokio::test] - async fn test_block_header_sync_complete_during_active_download() { - let mut manager = create_test_manager().await; - let (sender, _rx) = create_test_request_sender(); - - // Filter headers caught up to block tip, but target is higher (more headers coming) - manager.progress.update_current_height(1000); - manager.progress.update_target_height(2000); - manager.progress.update_block_header_tip_height(1000); - manager.set_state(SyncState::WaitForEvents); - - // BlockHeaderSyncComplete arrives but target not reached yet - let event = SyncEvent::BlockHeaderSyncComplete { - tip_height: 1000, - }; - let events = manager.handle_sync_event(&event, &sender).await.unwrap(); - - assert!(manager.block_headers_synced); - assert!(!events.iter().any(|e| matches!(e, SyncEvent::FilterHeadersSyncComplete { .. }))); + /// In-memory mode: a block-header store advanced the tip. Update the target + /// and flush any pending/retried cfheaders, but do NOT build batches from + /// storage (batches are driven by `BlockHeadersInMemory`). + pub(super) async fn handle_tip_update_in_memory( + &mut self, + tip_height: u32, + network: &Arc, + ) -> SyncResult> { + self.progress.update_block_header_tip_height(tip_height); + self.update_target_height(tip_height); + self.pipeline.send_pending(network).await?; + let mut events = Vec::new(); + if self.pipeline.is_complete() { + if let Some(event) = self.try_complete_sync() { + events.push(event); + } + } + Ok(events) } +} - #[tokio::test] - async fn test_on_disconnect() { - let mut manager = create_test_manager().await; - - // Set all fields that on_disconnect resets - manager.block_headers_synced = true; - manager.checkpoint_start_height = Some(500); - - manager.on_disconnect(); - - assert!(!manager.block_headers_synced); - assert!(manager.checkpoint_start_height.is_none()); - assert!(manager.pipeline.is_complete()); +impl std::fmt::Debug + for FilterHeadersManager +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FilterHeadersManager").field("progress", &self.progress).finish() } } diff --git a/dash-spv/src/sync/filter_headers/pipeline.rs b/dash-spv/src/sync/filter_headers/pipeline.rs index 309b28ca0..37c6a527e 100644 --- a/dash-spv/src/sync/filter_headers/pipeline.rs +++ b/dash-spv/src/sync/filter_headers/pipeline.rs @@ -1,36 +1,29 @@ //! CFHeaders pipeline implementation. //! //! Handles pipelined download of compact block filter headers (BIP 157/158). -//! Uses DownloadCoordinator for batch tracking with out-of-order buffering. +//! Tracks in-flight batches with simple pending/in-flight sets and out-of-order buffering. use dashcore::network::message::NetworkMessage; -use dashcore::network::message_filter::CFHeaders; +use dashcore::network::message_filter::{CFHeaders, GetCFHeaders}; use dashcore::BlockHash; use std::collections::HashMap; -use std::time::Duration; +use std::sync::Arc; use crate::error::{SyncError, SyncResult}; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::storage::BlockHeaderStorage; -use crate::sync::download_coordinator::{DownloadConfig, DownloadCoordinator}; +use crate::sync::download_coordinator::DownloadCoordinator; /// Batch size for filter header requests. const FILTER_HEADERS_BATCH_SIZE: u32 = 2000; -/// Maximum concurrent CFHeaders requests. -const MAX_CONCURRENT_CFHEADERS_REQUESTS: usize = 10; - -/// Timeout for CFHeaders requests (shorter for faster retry on multi-peer). -/// Timeout for CFHeaders requests. Single response but allow time for network latency. -const FILTER_HEADERS_TIMEOUT: Duration = Duration::from_secs(20); - /// Pipeline for downloading compact block filter headers. /// -/// Uses DownloadCoordinator for batch-level tracking (keyed by stop_hash), -/// with a HashMap buffer for out-of-order responses that need sequential processing. +/// Tracks batches by stop_hash using simple pending/in-flight sets, with a +/// HashMap buffer for out-of-order responses that need sequential processing. #[derive(Debug)] pub(super) struct FilterHeadersPipeline { - /// Core coordinator tracks batches by stop_hash. + /// Coordinator managing pending/in-flight batches (keyed by stop_hash). coordinator: DownloadCoordinator, /// Maps stop_hash -> start_height for each batch. batch_starts: HashMap, @@ -52,11 +45,7 @@ impl FilterHeadersPipeline { /// Create a new CFHeaders pipeline. pub(super) fn new() -> Self { Self { - coordinator: DownloadCoordinator::new( - DownloadConfig::default() - .with_max_concurrent(MAX_CONCURRENT_CFHEADERS_REQUESTS) - .with_timeout(FILTER_HEADERS_TIMEOUT), - ), + coordinator: DownloadCoordinator::new(), batch_starts: HashMap::new(), buffered: HashMap::new(), next_expected: 0, @@ -111,6 +100,80 @@ impl FilterHeadersPipeline { Ok(()) } + /// Initialize an empty, in-memory-driven pipeline (item #3). + /// + /// Unlike [`init`], this builds NO batches from storage. Batches are queued + /// incrementally via [`queue_batch`] as block headers land in memory, so + /// `getcfheaders` requests fan out across segments without waiting for the + /// sequential block-header store. `start_height` is the first filter-header + /// height (== `next_expected`); responses arriving out of order are buffered + /// and stitched in order exactly as in the storage path. + pub(super) fn init_in_memory(&mut self, start_height: u32, target_height: u32) { + self.coordinator.clear(); + self.batch_starts.clear(); + self.buffered.clear(); + self.next_expected = start_height; + self.target_height = target_height; + } + + /// Queue a single batch with an already-known stop hash (no storage lookup). + /// + /// Each validated block-header batch maps 1:1 to a `getcfheaders` request for + /// the same `[start_height, end_height]` range. Skips ranges already covered + /// (`end_height < next_expected`, e.g. re-emitted after a block-segment retry) + /// and dedupes batches still pending/in-flight by `stop_hash`. + pub(super) fn queue_batch(&mut self, start_height: u32, end_height: u32, stop_hash: BlockHash) { + if end_height < self.next_expected { + return; + } + if self.batch_starts.contains_key(&stop_hash) { + return; + } + if end_height > self.target_height { + self.target_height = end_height; + } + self.coordinator.enqueue([stop_hash]); + self.batch_starts.insert(stop_hash, start_height); + } + + /// In-memory-mode safety net: `true` if there is no pending/in-flight work + /// and the next expected batch isn't buffered, yet the target isn't reached + /// — i.e. a `BlockHeadersInMemory` event was dropped (lagged bus) and the + /// batch at `next_expected` must be recovered from storage. + pub(super) fn has_gap_at_next_expected(&self) -> bool { + self.coordinator.is_empty() + && !self.buffered.contains_key(&self.next_expected) + && self.target_height > 0 + && self.next_expected <= self.target_height + } + + /// Recover a dropped batch: queue `[next_expected, ..]` using the stop hash + /// from storage (which has the header by the time we've stalled). No-op if + /// the header isn't stored yet — the next tick retries. + pub(super) async fn reconcile_gap( + &mut self, + storage: &impl BlockHeaderStorage, + ) -> SyncResult<()> { + let start = self.next_expected; + let batch_end = (start + FILTER_HEADERS_BATCH_SIZE - 1).min(self.target_height); + if let Some(stop_hash) = storage.get_header(batch_end).await?.map(|h| *h.hash()) { + self.batch_starts.entry(stop_hash).or_insert(start); + self.coordinator.enqueue([stop_hash]); + tracing::warn!( + "FilterHeaders: reconciled dropped batch [{}..{}] from storage", + start, + batch_end + ); + } + Ok(()) + } + + /// Total filter-headers currently buffered (received out of order, awaiting + /// in-order verification). Used for the progress display. + pub(super) fn buffered_count(&self) -> u32 { + self.buffered.values().map(|c| c.filter_hashes.len() as u32).sum() + } + /// Get the next expected height for sequential processing. pub(super) fn next_expected(&self) -> u32 { self.next_expected @@ -163,8 +226,26 @@ impl FilterHeadersPipeline { Ok(()) } - /// Send pending requests using a RequestSender (synchronous). - pub(super) fn send_pending(&mut self, requests: &RequestSender) -> SyncResult { + /// Start the response timeout for a batch the router just put on the wire. + pub(super) fn mark_on_flight(&mut self, stop_hash: &BlockHash) { + self.coordinator.mark_on_flight(&[*stop_hash]); + } + + /// Send pending requests using the network manager (synchronous). + /// Re-queue GetCFHeaders batches whose response never arrived (peer died or + /// the message was lost), so the next `send_pending` re-issues them. + pub(super) fn handle_timeouts(&mut self, timeout: std::time::Duration) { + let timed_out = self.coordinator.take_timed_out(timeout); + if !timed_out.is_empty() { + tracing::warn!("FilterHeaders: re-queuing {} timed-out batch(es)", timed_out.len()); + self.coordinator.enqueue(timed_out); + } + } + + pub(super) async fn send_pending( + &mut self, + network: &Arc, + ) -> SyncResult { let count = self.coordinator.available_to_send(); if count == 0 { return Ok(0); @@ -181,7 +262,13 @@ impl FilterHeadersPipeline { ))); }; - requests.request_filter_headers(start_height, stop_hash)?; + network + .send(NetworkMessage::GetCFHeaders(GetCFHeaders { + filter_type: 0u8, + start_height, + stop_hash, + })) + .await; self.coordinator.mark_sent(&[stop_hash]); @@ -253,196 +340,4 @@ impl FilterHeadersPipeline { } ready } - - /// Re-enqueue timed out requests for retry. - pub(super) fn handle_timeouts(&mut self) { - for stop_hash in self.coordinator.check_timeouts() { - self.coordinator.enqueue_retry(stop_hash); - } - } -} - -#[cfg(test)] -mod tests { - use dashcore_hashes::Hash; - - use super::*; - - #[test] - fn test_cfheaders_pipeline_new() { - let pipeline = FilterHeadersPipeline::new(); - assert!(pipeline.is_complete()); - } - - #[test] - fn test_match_response_empty() { - let pipeline = FilterHeadersPipeline::new(); - - let empty_cfheaders = CFHeaders { - filter_type: 0, - stop_hash: dashcore::BlockHash::all_zeros(), - previous_filter_header: dashcore::hash_types::FilterHeader::all_zeros(), - filter_hashes: vec![], - }; - - // Empty response should return None - assert!(pipeline.match_response(&NetworkMessage::CFHeaders(empty_cfheaders)).is_none()); - } - - #[test] - fn test_match_response_wrong_message() { - let pipeline = FilterHeadersPipeline::new(); - - // Wrong message type should return None - assert!(pipeline.match_response(&NetworkMessage::Verack).is_none()); - } - - #[test] - fn test_receive_in_order() { - use dashcore::hash_types::FilterHash; - - let mut pipeline = FilterHeadersPipeline::new(); - pipeline.next_expected = 1; - pipeline.target_height = 100; - - let stop_hash = BlockHash::all_zeros(); - - // Mark batch as in-flight (by stop_hash) - pipeline.coordinator.mark_sent(&[stop_hash]); - pipeline.batch_starts.insert(stop_hash, 1); - - let cfheaders = CFHeaders { - filter_type: 0, - stop_hash, - previous_filter_header: dashcore::hash_types::FilterHeader::all_zeros(), - filter_hashes: vec![FilterHash::all_zeros()], - }; - - // Should return data immediately - let result = pipeline.receive(1, cfheaders.clone()); - assert!(result.is_some()); - } - - #[test] - fn test_receive_out_of_order() { - use dashcore::hash_types::FilterHash; - - let mut pipeline = FilterHeadersPipeline::new(); - pipeline.next_expected = 1; - pipeline.target_height = 4000; - - let stop_hash = BlockHash::all_zeros(); - - // Mark batch as in-flight (by stop_hash) - pipeline.coordinator.mark_sent(&[stop_hash]); - pipeline.batch_starts.insert(stop_hash, 2000); - - let cfheaders = CFHeaders { - filter_type: 0, - stop_hash, - previous_filter_header: dashcore::hash_types::FilterHeader::all_zeros(), - filter_hashes: vec![FilterHash::all_zeros()], - }; - - // Should buffer (out of order) - let result = pipeline.receive(2000, cfheaders); - assert!(result.is_none()); - assert_eq!(pipeline.buffered.len(), 1); - } - - #[test] - fn test_advance_returns_buffered() { - use dashcore::hash_types::FilterHash; - - let mut pipeline = FilterHeadersPipeline::new(); - pipeline.next_expected = 1; - pipeline.target_height = 4000; - - // Buffer a response at height 2000 - let cfheaders = CFHeaders { - filter_type: 0, - stop_hash: BlockHash::all_zeros(), - previous_filter_header: dashcore::hash_types::FilterHeader::all_zeros(), - filter_hashes: vec![FilterHash::all_zeros()], - }; - pipeline.buffered.insert(2000, cfheaders); - - // Advance to 2000 - let ready = pipeline.advance(1999); - assert_eq!(ready.len(), 1); - assert_eq!(ready[0].0, 2000); - assert_eq!(pipeline.buffered.len(), 0); - } - - #[test] - fn test_handle_timeouts_basic_retry() { - use std::time::Duration; - - let mut pipeline = FilterHeadersPipeline { - coordinator: DownloadCoordinator::new( - DownloadConfig::default().with_timeout(Duration::from_millis(1)), - ), - batch_starts: HashMap::new(), - buffered: HashMap::new(), - next_expected: 1, - target_height: 2000, - }; - - let stop_hash = BlockHash::all_zeros(); - pipeline.coordinator.mark_sent(&[stop_hash]); - pipeline.batch_starts.insert(stop_hash, 1); - - std::thread::sleep(Duration::from_millis(5)); - - pipeline.handle_timeouts(); - assert_eq!(pipeline.coordinator.pending_count(), 1); - } - - #[test] - fn test_send_pending_errors_on_missing_batch_starts() { - let mut pipeline = FilterHeadersPipeline::new(); - pipeline.next_expected = 1; - pipeline.target_height = 2000; - - let hash_without_entry = BlockHash::from_byte_array([0x02; 32]); - - // Enqueue a stop_hash without a corresponding batch_starts entry - pipeline.coordinator.enqueue([hash_without_entry]); - - let (tx, _rx) = tokio::sync::mpsc::unbounded_channel(); - let requests = RequestSender::new(tx); - - let err = pipeline.send_pending(&requests).unwrap_err(); - assert!(matches!(err, SyncError::InvalidState(_))); - } - - #[test] - fn test_handle_timeouts_multiple_batches() { - use std::time::Duration; - - let mut pipeline = FilterHeadersPipeline { - coordinator: DownloadCoordinator::new( - DownloadConfig::default().with_timeout(Duration::from_millis(1)), - ), - batch_starts: HashMap::new(), - buffered: HashMap::new(), - next_expected: 1, - target_height: 4000, - }; - - let hash1 = BlockHash::from_byte_array([0x01; 32]); - let hash2 = BlockHash::from_byte_array([0x02; 32]); - - pipeline.coordinator.mark_sent(&[hash1, hash2]); - pipeline.batch_starts.insert(hash1, 1); - pipeline.batch_starts.insert(hash2, 2001); - - std::thread::sleep(Duration::from_millis(5)); - - pipeline.handle_timeouts(); - // Both batches re-queued - assert_eq!(pipeline.coordinator.pending_count(), 2); - assert!(pipeline.batch_starts.contains_key(&hash1)); - assert!(pipeline.batch_starts.contains_key(&hash2)); - } } diff --git a/dash-spv/src/sync/filter_headers/progress.rs b/dash-spv/src/sync/filter_headers/progress.rs index 1ec82679d..9343e369a 100644 --- a/dash-spv/src/sync/filter_headers/progress.rs +++ b/dash-spv/src/sync/filter_headers/progress.rs @@ -14,6 +14,12 @@ pub struct FilterHeadersProgress { target_height: u32, /// The tip height of the block-header storage (the download limit for filter headers). block_header_tip_height: u32, + /// Number of filter-headers currently buffered in the pipeline (downloaded + /// out of order, awaiting in-order verification). Mirrors block-header + /// progress: `current_height` is only the contiguous verified prefix, so the + /// display adds `buffered` to show realistic parallel-download progress + /// without jumping to the tip (this is a count, not the frontier height). + buffered: u32, /// Number of filter-headers processed (stored) in the current sync session. processed: u32, /// The last time a filter-header was stored to disk or the last manager state change. @@ -27,6 +33,7 @@ impl Default for FilterHeadersProgress { current_height: 0, target_height: 0, block_header_tip_height: 0, + buffered: 0, processed: 0, last_activity: Instant::now(), } @@ -44,6 +51,26 @@ impl FilterHeadersProgress { } /// Number of filter-headers processed (stored) in the current sync session. + /// Verified prefix + headers downloaded but still buffered awaiting in-order + /// verification — i.e. how far the DOWNLOAD has got, which is what the progress + /// line and the progress bar show. Headers come down in parallel, so a display + /// counting only the verified prefix sits still and then jumps. + /// + /// For display only. Anything that must not act on unverified data wants + /// `current_height()` (the verified tip). + pub fn effective_height(&self) -> u32 { + (self.current_height + self.buffered).min(self.target_height) + } + + /// Download progress (0.0..=1.0) — `effective_height` over the target. What the + /// `Display` line prints, so a bar fed from it reads the same as the log. + pub fn download_percentage(&self) -> f64 { + if self.target_height == 0 { + return 0.0; + } + (self.effective_height() as f64 / self.target_height as f64).min(1.0) + } + pub fn processed(&self) -> u32 { self.processed } @@ -65,6 +92,17 @@ impl FilterHeadersProgress { self.bump_last_activity(); } + /// Update the count of filter-headers buffered in the pipeline (downloaded, + /// awaiting in-order verification). Drives the realistic display percentage. + pub fn update_buffered(&mut self, count: u32) { + self.buffered = count; + } + + /// Number of filter-headers currently buffered awaiting verification. + pub fn buffered(&self) -> u32 { + self.buffered + } + /// Update the target height (peer's best height, for progress display). /// Only updates if the new height is greater than the current target (monotonic increase). pub fn update_target_height(&mut self, height: u32) { @@ -94,15 +132,16 @@ impl FilterHeadersProgress { impl fmt::Display for FilterHeadersProgress { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let pct = self.percentage() * 100.0; write!( f, - "{:?} {}/{} ({:.1}%) processed: {}, last_activity: {}s", + "{:?} {}/{} ({:.1}%) verified: {}, processed: {}, buffered: {}, last_activity: {}s", self.state, - self.current_height, + self.effective_height(), self.target_height, - pct, + self.download_percentage() * 100.0, + self.current_height, self.processed, + self.buffered, self.last_activity.elapsed().as_secs() ) } @@ -112,6 +151,18 @@ impl ProgressPercentage for FilterHeadersProgress { fn target_height(&self) -> u32 { self.target_height } + /// The VERIFIED tip: the contiguous prefix of the chain checked in order and + /// written to storage. + /// + /// Deliberately NOT the effective height (see [`effective_height`]): every + /// caller of this is logic — when filter-header sync is complete, where to + /// resume requesting, and the tip published to `FiltersManager`, which decides + /// from it which filters it may verify. Counting buffered-but-unverified + /// headers here wedged the sync: filters were told the chain reached the tip, + /// tried to verify filters against headers that did not exist yet, and stopped + /// dead. + /// + /// [`effective_height`]: FilterHeadersProgress::effective_height fn current_height(&self) -> u32 { self.current_height } diff --git a/dash-spv/src/sync/filter_headers/sync_manager.rs b/dash-spv/src/sync/filter_headers/sync_manager.rs index eae554d57..8366bd7c8 100644 --- a/dash-spv/src/sync/filter_headers/sync_manager.rs +++ b/dash-spv/src/sync/filter_headers/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network::{Message, MessageType, RequestSender}; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage}; use crate::sync::filter_headers::pipeline::FilterHeadersPipeline; use crate::sync::progress::ProgressPercentage; @@ -8,6 +8,9 @@ use crate::sync::{ }; use crate::SyncError; use async_trait::async_trait; +use dashcore::network::message::NetworkMessage; +use std::net::SocketAddr; +use std::sync::Arc; #[async_trait] impl SyncManager for FilterHeadersManager { @@ -27,8 +30,14 @@ impl SyncManager for FilterHeade self.progress.update_target_height(height); } - fn wanted_message_types(&self) -> &'static [MessageType] { - &[MessageType::CFHeaders] + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { + &[crate::network2::MessageType::CfHeaders] + } + + fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { + if let crate::network2::RequestKey::CfHeaders(stop_hash) = key { + self.pipeline.mark_on_flight(stop_hash); + } } fn on_disconnect(&mut self) { @@ -39,11 +48,12 @@ impl SyncManager for FilterHeade async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult> { // Match response to get start height - let Some((start_height, cfheaders)) = self.pipeline.match_response(msg.inner()) else { + let Some((start_height, cfheaders)) = self.pipeline.match_response(&msg) else { if self.pipeline.is_complete() { if let Some(event) = self.try_complete_sync() { return Ok(vec![event]); @@ -57,7 +67,8 @@ impl SyncManager for FilterHeade // Try to receive (may buffer if out of order) if let Some(data) = self.pipeline.receive(start_height, cfheaders) { // In order - process immediately - let count = self.process_cfheaders(&data, start_height).await?; + let headers = self.process_cfheaders(&data, start_height).await?; + let count = headers.len() as u32; if count == 0 { return Err(SyncError::Network("CFHeaders batch contained no headers".to_string())); } @@ -81,13 +92,15 @@ impl SyncManager for FilterHeade start_height: batch_start, end_height: batch_end, tip_height: self.progress.current_height(), + headers, }); // Process buffered responses (including any returned by first advance) while !ready_batches.is_empty() { // Take ownership and process each batch for (height, data) in std::mem::take(&mut ready_batches) { - let count = self.process_cfheaders(&data, height).await?; + let headers = self.process_cfheaders(&data, height).await?; + let count = headers.len() as u32; if count == 0 { return Err(SyncError::Network( "CFHeaders batch contained no headers".to_string(), @@ -103,6 +116,7 @@ impl SyncManager for FilterHeade start_height: height, end_height: height + count.saturating_sub(1), tip_height: self.progress.current_height(), + headers, }); } } @@ -114,8 +128,12 @@ impl SyncManager for FilterHeade ); } + // Refresh the buffered count for the progress display (headers downloaded + // out of order, awaiting in-order verification). + self.progress.update_buffered(self.pipeline.buffered_count()); + // Send more requests - self.pipeline.send_pending(requests)?; + self.pipeline.send_pending(network).await?; if self.pipeline.is_complete() { if let Some(event) = self.try_complete_sync() { @@ -129,28 +147,55 @@ impl SyncManager for FilterHeade async fn handle_sync_event( &mut self, event: &SyncEvent, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { match event { SyncEvent::BlockHeaderSyncComplete { tip_height, } => { self.block_headers_synced = true; - self.handle_new_headers(*tip_height, requests).await + self.ensure_mode().await?; + if self.in_memory_driven == Some(true) { + self.handle_tip_update_in_memory(*tip_height, network).await + } else { + self.handle_new_headers(*tip_height, network).await + } + } + SyncEvent::BlockHeadersInMemory { + start_height, + end_height, + stop_hash, + } => { + self.handle_headers_in_memory(*start_height, *end_height, *stop_hash, network).await } SyncEvent::BlockHeadersStored { tip_height, - } => self.handle_new_headers(*tip_height, requests).await, + } => { + self.ensure_mode().await?; + if self.in_memory_driven == Some(true) { + self.handle_tip_update_in_memory(*tip_height, network).await + } else { + self.handle_new_headers(*tip_height, network).await + } + } _ => Ok(vec![]), } } - async fn tick(&mut self, requests: &RequestSender) -> SyncResult> { - // Handle timed out requests (re-queues them for retry) - self.pipeline.handle_timeouts(); + async fn tick(&mut self, network: &Arc) -> SyncResult> { + // Re-issue any timed-out requests, then send pending. + self.pipeline.handle_timeouts(std::time::Duration::from_secs(20)); + + // In-memory-mode safety net: if a BlockHeadersInMemory event was dropped + // (lagged bus), the pipeline stalls with a gap at next_expected. Recover + // the missing batch from storage (the header is stored by now). + if self.in_memory_driven == Some(true) && self.pipeline.has_gap_at_next_expected() { + let header_storage = self.header_storage.read().await; + self.pipeline.reconcile_gap(&*header_storage).await?; + drop(header_storage); + } - // Send pending requests (including retries) - self.pipeline.send_pending(requests)?; + self.pipeline.send_pending(network).await?; Ok(vec![]) } diff --git a/dash-spv/src/sync/filters/batch.rs b/dash-spv/src/sync/filters/batch.rs index 386e0d16e..80a5aae6d 100644 --- a/dash-spv/src/sync/filters/batch.rs +++ b/dash-spv/src/sync/filters/batch.rs @@ -2,6 +2,7 @@ use dashcore::bip158::BlockFilter; use dashcore::ScriptBuf; use key_wallet_manager::{FilterMatchKey, WalletId}; use std::collections::{BTreeSet, HashMap, HashSet}; +use std::sync::Arc; /// A completed batch of compact block filters ready for verification. /// @@ -16,14 +17,17 @@ pub(super) struct FiltersBatch { end_height: u32, /// Filters of this batch. filters: HashMap, + /// Cached `Arc` snapshot of `filters` as an indexable Vec, built once and + /// shared (not cloned) across every bulk rescan pass, so a script fanned out + /// to many open batches doesn't re-clone their filter sets each generation. + /// Invalidated whenever `filters` is mutated. + filters_arc: Option>>, /// Whether this batch was verified already (loaded from storage). verified: bool, /// Whether this batch was scanned already. scanned: bool, /// Number of blocks still being downloaded for this batch. pending_blocks: u32, - /// Whether rescan has been completed for this batch. - rescan_complete: bool, /// Wallets that were behind for this batch's height range at scan time and /// therefore need their `synced_height` advanced when the batch commits. /// Already-synced wallets must not be touched. @@ -32,6 +36,13 @@ pub(super) struct FiltersBatch { /// need rescan, attributed per wallet so we can rerun matching only /// against the wallet that produced each new script. collected_scripts: HashMap>, + /// How many entries of the manager's append-only derived-script log this batch + /// has been matched against. Scanning runs ahead of the commit frontier (that + /// is what overlaps matching with the download), so a script derived by a lower + /// batch can reach a batch that was already scanned — the cursor records how + /// far each batch has caught up, and it may only commit once it has seen the + /// whole log. + deferred_cursor: usize, } impl FiltersBatch { @@ -45,12 +56,13 @@ impl FiltersBatch { start_height, end_height, filters, + filters_arc: None, verified: false, scanned: false, pending_blocks: 0, - rescan_complete: false, scanned_wallets: BTreeSet::new(), collected_scripts: HashMap::new(), + deferred_cursor: 0, } } /// Start height of this batch (inclusive). @@ -67,8 +79,33 @@ impl FiltersBatch { } /// Mutable reference to the loaded filters map of this batch. pub(super) fn filters_mut(&mut self) -> &mut HashMap { + // Any mutation invalidates the shared snapshot. + self.filters_arc = None; &mut self.filters } + /// Shared `Arc` snapshot of this batch's filters as an indexable Vec, built + /// once on first use and reused across every rescan (no per-pass cloning). + /// + /// Building it MOVES the filters out of the `HashMap`: once a batch is + /// scanned, every remaining consumer (the head rescan and the bulk lookahead) + /// wants exactly this shared snapshot, so keeping the map as well would hold + /// a second full copy of every filter — with ~1000 batches open at the peak + /// that second copy costs GBs. `filters()` is therefore empty after the first + /// call here, which is safe because a batch's filters are all loaded (during + /// the sequential store) strictly before it is scanned. + pub(super) fn filters_arc(&mut self) -> Arc> { + if self.filters_arc.is_none() { + let v: Vec<(FilterMatchKey, BlockFilter)> = std::mem::take(&mut self.filters) + .into_iter() + .collect::>(); + self.filters_arc = Some(Arc::new(v)); + } + self.filters_arc.clone().expect("just set") + } + /// Bytes held by the sealed `Arc` snapshot (0 if it has not been built). + pub(super) fn filters_arc_bytes(&self) -> usize { + self.filters_arc.as_ref().map(|a| a.iter().map(|(_, f)| f.content.len()).sum()).unwrap_or(0) + } /// Returns whether this batch is verified (filters verified against their headers). pub(super) fn verified(&self) -> bool { self.verified @@ -98,14 +135,6 @@ impl FiltersBatch { self.pending_blocks = self.pending_blocks.saturating_sub(1); self.pending_blocks } - /// Returns whether rescan has been completed for this batch. - pub(super) fn rescan_complete(&self) -> bool { - self.rescan_complete - } - /// Mark rescan as complete for this batch. - pub(super) fn mark_rescan_complete(&mut self) { - self.rescan_complete = true; - } /// Add scriptPubKeys discovered during block processing for later rescan. pub(super) fn add_scripts_for_wallet( &mut self, @@ -118,6 +147,14 @@ impl FiltersBatch { pub(super) fn take_collected_scripts(&mut self) -> HashMap> { std::mem::take(&mut self.collected_scripts) } + /// How many entries of the derived-script log this batch has been matched against. + pub(super) fn deferred_cursor(&self) -> usize { + self.deferred_cursor + } + /// Advance this batch's cursor to `n` entries of the derived-script log. + pub(super) fn set_deferred_cursor(&mut self, n: usize) { + self.deferred_cursor = n; + } /// Record the set of wallets that were behind for this batch at scan time. pub(super) fn set_scanned_wallets(&mut self, wallets: BTreeSet) { self.scanned_wallets = wallets; diff --git a/dash-spv/src/sync/filters/batch_tracker.rs b/dash-spv/src/sync/filters/batch_tracker.rs index d67a74653..c657bb1e4 100644 --- a/dash-spv/src/sync/filters/batch_tracker.rs +++ b/dash-spv/src/sync/filters/batch_tracker.rs @@ -52,6 +52,11 @@ impl BatchTracker { self.end_height } /// Number of filters received in this batch. + /// TEMPORARY: filters buffered in this (possibly incomplete) batch. + pub(super) fn buffered(&self) -> (usize, usize) { + (self.filters.len(), self.filters.values().map(|f| f.content.len()).sum()) + } + pub(super) fn received(&self) -> u32 { self.received.len() as u32 } diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index 58192af13..cdfb12538 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -14,7 +14,7 @@ use super::batch::FiltersBatch; use super::block_match_tracker::{BlockMatchTracker, BlockTrackResult}; use super::pipeline::FiltersPipeline; use crate::error::SyncResult; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage, FilterStorage}; use crate::sync::filters::util::get_prev_filter_header; use crate::sync::{FiltersProgress, SyncEvent, SyncManager, SyncState}; @@ -23,12 +23,186 @@ use crate::validation::{FilterValidationInput, FilterValidator, Validator}; use crate::sync::progress::ProgressPercentage; use dashcore::hash_types::FilterHeader; use key_wallet_manager::WalletInterface; -use key_wallet_manager::{check_compact_filters_for_elements, FilterMatchKey, WalletId}; +use key_wallet_manager::{FilterMatchKey, WalletId}; use tokio::sync::RwLock; /// Batch size for processing filters. const BATCH_PROCESSING_SIZE: u32 = 5000; +/// Maximum shards a batch's filters are split into for parallel matching. +const MATCH_SHARD_CAP: usize = 16; + +/// Number of shards to split filter matching into, bounded by available cores. +fn match_shards() -> usize { + std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).min(MATCH_SHARD_CAP) +} + +/// Match a batch's filters against `scripts`/`elements`, sharded across tokio's +/// blocking pool. BIP158 GCS matching is CPU-bound and independent per filter, +/// so each shard runs `match_any` over a slice concurrently. Runs on tokio's +/// blocking pool (not rayon's global pool, which the block-header hashing uses). +/// +/// `filters` is passed pre-snapshotted into an `Arc` so a caller matching +/// several query sets against the same batch (the rescan path) shares one copy. +async fn match_filters_parallel( + filters: Arc>, + scripts: Vec, + elements: Vec>, + min_height: u32, +) -> BTreeSet { + let n = filters.len(); + if n == 0 { + return BTreeSet::new(); + } + + let shards = match_shards().min(n); + let chunk = n.div_ceil(shards); + let scripts = Arc::new(scripts); + let elements = Arc::new(elements); + + let mut handles = Vec::with_capacity(shards); + for start in (0..n).step_by(chunk) { + let filters = filters.clone(); + let scripts = scripts.clone(); + let elements = elements.clone(); + let end = (start + chunk).min(n); + handles.push(tokio::task::spawn_blocking(move || { + // The query borrows the scripts/elements, so build it per shard. + let mut query: FilterQuery = scripts.iter().map(|s| s.as_bytes()).collect(); + for element in elements.iter() { + query.push(element); + } + let mut out = BTreeSet::new(); + for (key, filter) in &filters[start..end] { + if key.height() <= min_height { + continue; + } + match filter.match_any(key.hash(), &query) { + Ok(true) => { + out.insert(key.clone()); + } + Ok(false) => {} + Err(e) => { + tracing::warn!( + "filter match_any error at height {}: {}; treating as non-match", + key.height(), + e + ); + } + } + } + out + })); + } + + let mut result = BTreeSet::new(); + for handle in handles { + if let Ok(set) = handle.await { + result.extend(set); + } + } + result +} + +/// Like [`match_filters_parallel`] but over SEVERAL batches' filter snapshots at +/// once, sharded across the flattened filter space. The per-batch `Arc`s are +/// shared into the shards (only the pointer is cloned), so a bulk rescan fanning +/// one script set across many open batches never re-clones their filters — the +/// memory blow-up that a single combined `Vec` would cause. +async fn match_filters_multi( + batches: Vec>>, + scripts: Vec, + elements: Vec>, + min_height: u32, +) -> BTreeSet { + let total: usize = batches.iter().map(|b| b.len()).sum(); + if total == 0 { + return BTreeSet::new(); + } + let shards = match_shards().min(total); + let chunk = total.div_ceil(shards); + let scripts = Arc::new(scripts); + let elements = Arc::new(elements); + + // Carve the flattened [0..total) filter space into `shards` contiguous + // chunks, each expressed as (batch_index, start, end) sub-ranges. + let mut shard_ranges: Vec> = vec![Vec::new(); shards]; + let mut s = 0usize; + let mut fill = 0usize; + for (bi, b) in batches.iter().enumerate() { + let blen = b.len(); + let mut off = 0; + while off < blen { + if s < shards - 1 && fill >= chunk { + s += 1; + fill = 0; + } + let take = (chunk - fill).min(blen - off); + shard_ranges[s].push((bi, off, off + take)); + off += take; + fill += take; + } + } + + let mut handles = Vec::with_capacity(shards); + for ranges in shard_ranges { + if ranges.is_empty() { + continue; + } + let batches = batches.clone(); // Vec — pointer clones only + let scripts = scripts.clone(); + let elements = elements.clone(); + handles.push(tokio::task::spawn_blocking(move || { + let mut query: FilterQuery = scripts.iter().map(|s| s.as_bytes()).collect(); + for element in elements.iter() { + query.push(element); + } + let mut out = BTreeSet::new(); + for (bi, start, end) in ranges { + for (key, filter) in &batches[bi][start..end] { + if key.height() <= min_height { + continue; + } + match filter.match_any(key.hash(), &query) { + Ok(true) => { + out.insert(key.clone()); + } + Ok(false) => {} + Err(e) => { + tracing::warn!( + "filter match_any error at height {}: {}; treating as non-match", + key.height(), + e + ); + } + } + } + } + out + })); + } + + let mut result = BTreeSet::new(); + for handle in handles { + if let Ok(set) = handle.await { + result.extend(set); + } + } + result +} + +/// Folds the elapsed time into `P_SCAN` on drop, so `scan_batch`'s several early +/// returns all get accounted for. +struct ScanGuard(std::time::Instant); +impl Drop for ScanGuard { + fn drop(&mut self) { + crate::timer::P_SCAN.add(self.0.elapsed()); + } +} +fn scopeguard_scan(t: std::time::Instant) -> ScanGuard { + ScanGuard(t) +} + /// Snapshot of a behind wallet's compact-filter query inputs for a batch scan. struct WalletScanState { /// The wallet these inputs belong to. @@ -42,9 +216,6 @@ struct WalletScanState { elements: Vec>, } -/// Maximum number of batches to scan ahead while waiting for blocks. -const MAX_LOOKAHEAD_BATCHES: usize = 3; - /// Filters manager for downloading and matching compact block filters. /// /// Generic over: @@ -73,7 +244,17 @@ pub struct FiltersManager< /// Completed batches waiting for verification and storage. pub(super) pending_batches: BTreeSet, /// Next batch start height to store (for filter verification/storage). - next_batch_to_store: u32, + /// Contiguous watermark: every filter at or below this height is on disk. + /// Batches are stored as soon as they are complete and verified, in whatever + /// order they arrive, so this walks forward over `stored_ends` as the holes + /// fill. Scanning gates on it, and it must never claim a range with a hole. + stored_watermark: u32, + /// Batches written out of order, keyed by start -> end, waiting for the + /// watermark to reach them. Just bookkeeping — their filters are already on + /// disk, not in memory. + stored_ends: BTreeMap, + /// The next START height the watermark waits for. + next_store_start: u32, // === Multi-batch processing state === /// Active batches being processed (keyed by start_height). @@ -84,6 +265,23 @@ pub struct FiltersManager< /// `BlockProcessed` and the per-wallet record of which wallets already /// have a given processed block applied. pub(super) tracker: BlockMatchTracker, + /// Append-only log of scriptPubKeys derived (gap-limit) while processing + /// matched blocks, tagged by owning wallet. Once per generation — when the + /// block pipeline drains — the manager bulk-matches the unseen tail of this + /// log against ALL open batches at once (`bulk_rescan`), so a whole wave of + /// matched blocks is requested and downloaded in parallel instead of one + /// round-trip per batch behind the sequential commit. A batch commits only + /// once its `deferred_cursor` reaches the log length. + pub(super) deferred: Vec<(WalletId, ScriptBuf)>, + /// Dedup guard for `deferred` so a script derived by several blocks is + /// logged (and thus rescanned) only once. + pub(super) deferred_seen: HashSet<(WalletId, ScriptBuf)>, + /// Verified filter headers pushed over by `FilterHeadersManager`, keyed by + /// height, held only until the filters they authenticate have been verified + /// and stored (then pruned). Filters are requested as far as the BLOCK headers + /// reach, so they arrive around the same time as these do — keeping them here + /// means the verification never has to read back what we were just handed. + pub(super) filter_headers: BTreeMap, } impl @@ -123,11 +321,24 @@ impl, ) -> SyncResult> { debug_assert!(self.is_idle(), "manager should have no in-flight state on start"); @@ -213,12 +426,13 @@ impl self.progress.filter_header_tip_height() { @@ -252,11 +466,14 @@ impl = self + .pending_batches + .iter() + .filter(|b| b.end_height() <= self.progress.filter_header_tip_height()) + .map(|b| b.start_height()) + .collect(); - let mut batch = self.pending_batches.pop_first().unwrap(); + for start in storable { + // Filters can be verified only against filter headers whose chain has + // been checked in order (an unverified header authenticates nothing — a + // peer could serve a filter that omits our transaction). Filters run + // ahead of that chain by design, so a batch whose headers are not there + // yet simply waits in `pending_batches` for a later pass. + // `FiltersBatch` orders and compares by start height, so the set can be + // probed with an empty batch carrying just that key. + let Some(mut batch) = + self.pending_batches.take(&FiltersBatch::new(start, start, HashMap::new())) + else { + continue; + }; + let (bs, be) = (batch.start_height(), batch.end_height()); tracing::debug!( "Storing filter batch {} to {} ({} filters)", @@ -347,25 +583,38 @@ impl = filter_headers - .into_iter() - .enumerate() - .map(|(idx, header)| (batch.start_height() + idx as u32, header)) + // The headers for this range were handed to us on the + // `FilterHeadersStored` event and kept in memory, so verification + // normally needs no storage read at all. Fall back to storage only + // for ranges verified before this run (a resumed sync), where the + // event fired in a previous process. + let mut filter_headers_map: HashMap = (bs..=be) + .filter_map(|h| self.filter_headers.get(&h).map(|fh| (h, *fh))) .collect(); + if filter_headers_map.len() != (be - bs + 1) as usize { + let loaded = self + .filter_header_storage + .read() + .await + .load_filter_headers(bs..be + 1) + .await?; + for (idx, header) in loaded.into_iter().enumerate() { + filter_headers_map.entry(bs + idx as u32).or_insert(header); + } + } - let filter_header_storage = self.filter_header_storage.read().await; + // The chain link before this batch: also usually in memory (it is + // the last header of the previous batch's event). let prev_filter_header = - get_prev_filter_header(&*filter_header_storage, batch.start_height()).await?; - drop(filter_header_storage); + match bs.checked_sub(1).and_then(|h| self.filter_headers.get(&h).copied()) { + Some(h) => h, + None => { + let fhs = self.filter_header_storage.read().await; + get_prev_filter_header(&*fhs, bs).await? + } + }; + let t_verify = std::time::Instant::now(); let validator = FilterValidator::new(); let validation_input = FilterValidationInput { filters: batch.filters(), @@ -373,22 +622,37 @@ impl= be); } // === Load filters into all active batches that overlap === for active_batch in self.active_batches.values_mut() { + // A scanned batch has already sealed its filters into the shared + // snapshot (and dropped the map); it needs no more loading — all + // of its filters were stored before it could be scanned. + if active_batch.scanned() { + continue; + } if batch.start_height() <= active_batch.end_height() && batch.end_height() >= active_batch.start_height() { @@ -415,9 +679,23 @@ impl SyncResult> { let mut events = Vec::new(); - // Phase 1: Commit completed batches in order + // MEM (throttled): what the filters manager is actually holding. Sums the + // real filter bytes rather than counting structures, so the numbers can be + // compared against the process RSS directly. + { + static LAST: std::sync::OnceLock> = + std::sync::OnceLock::new(); + let m = LAST.get_or_init(|| std::sync::Mutex::new(std::time::Instant::now())); + if let Ok(mut last) = m.lock() { + if last.elapsed() >= std::time::Duration::from_secs(2) { + *last = std::time::Instant::now(); + let pend_bytes: usize = self + .pending_batches + .iter() + .flat_map(|b| b.filters().values()) + .map(|f| f.content.len()) + .sum(); + let act_bytes: usize = self + .active_batches + .values() + .flat_map(|b| b.filters().values()) + .map(|f| f.content.len()) + .sum(); + let arc_bytes: usize = + self.active_batches.values().map(|b| b.filters_arc_bytes()).sum(); + let (trk_batches, trk_filters, trk_bytes) = + self.filter_pipeline.buffered_in_trackers(); + tracing::info!( + target: "dash_spv::sync::filters", + "MEM incomplete={} batches ({} filters, {:.0} MB) | pending={} batches ({:.0} MB) | active={} batches (map {:.0} MB + snapshot {:.0} MB) | fheader_cache={} ({:.0} MB)", + trk_batches, + trk_filters, + trk_bytes as f64 / 1e6, + self.pending_batches.len(), + pend_bytes as f64 / 1e6, + self.active_batches.len(), + act_bytes as f64 / 1e6, + arc_bytes as f64 / 1e6, + self.filter_headers.len(), + (self.filter_headers.len() * 36) as f64 / 1e6, + ); + } + } + } + + // Phase 0: fold the scripts that freshly processed blocks derived into the + // shared append-only log. + self.fold_derived_scripts(); + + // Phase 1: once the block pipeline drains, match the new generation of + // derived scripts against every open batch at once and request that whole + // wave of blocks together (they download in parallel). + events.extend(self.bulk_rescan().await?); + + // Phase 2: commit whatever is settled, in order. events.extend(self.try_commit_batches().await?); - // Phase 2: Scan any ready batches where filters are available + // Phase 3: scan every batch whose filters are ready, in ONE parallel pass. events.extend(self.scan_ready_batches().await?); - // Phase 3: Commit newly scanned batches that are ready - // (avoids a one-tick delay between scan and commit for small batches) + // Phase 4: a fresh scan may have drained the pipeline; take another + // generation, then commit what that settles. + events.extend(self.bulk_rescan().await?); events.extend(self.try_commit_batches().await?); - // Phase 4: Create lookahead batches up to MAX_LOOKAHEAD_BATCHES + // Phase 5: extend the lookahead up to the available filter-header tip. events.extend(self.try_create_lookahead_batches().await?); // If no active batches and all filters downloaded, emit FiltersSyncComplete. @@ -472,63 +819,219 @@ impl SyncResult> { + /// Move scripts derived while processing matched blocks into the shared + /// append-only log, deduplicated, so the next `bulk_rescan` fans them out. + fn fold_derived_scripts(&mut self) { + let drained: Vec>> = self + .active_batches + .values_mut() + .filter_map(|b| { + let m = b.take_collected_scripts(); + (!m.is_empty()).then_some(m) + }) + .collect(); + for m in drained { + for (wallet_id, scripts) in m { + for s in scripts { + if self.deferred_seen.insert((wallet_id, s.clone())) { + self.deferred.push((wallet_id, s)); + } + } + } + } + } + + /// Blocks still downloading across all open batches. + fn total_pending_blocks(&self) -> u32 { + self.active_batches.values().map(|b| b.pending_blocks()).sum() + } + + /// Lookahead ahead of the commit frontier: once the block pipeline drains, + /// match the unseen tail of the derived-script log against ALL open batches in + /// one pass per wallet, so a whole generation of gap-limit matches is found and + /// its blocks requested together instead of one round-trip at a time as the + /// frontier reaches each batch. + /// + /// Gated on a drained pipeline so this full-filter-set match runs about once + /// per generation, not once per processed block (unguarded, that was a 300x + /// blowup in match calls and a 2.5x sync regression). Every (script, batch) pair + /// is still matched exactly once — the per-batch cursor guarantees it. + async fn bulk_rescan(&mut self) -> SyncResult> { + let t_bulk = std::time::Instant::now(); let mut events = Vec::new(); + let log_len = self.deferred.len(); + if log_len == 0 || self.total_pending_blocks() > 0 { + return Ok(events); + } - while let Some((&batch_start, batch)) = self.active_batches.first_key_value() { - // Check if batch was scanned - can't commit until scanned - if !batch.scanned() { - break; + let starts: Vec = self + .active_batches + .iter() + .filter(|(_, b)| b.scanned() && b.deferred_cursor() < log_len) + .map(|(&s, _)| s) + .collect(); + if starts.is_empty() { + return Ok(events); + } + + { + // Lowest cursor among the laggards: the tail every open batch still has + // to see. Batches already past part of it are re-tested against the + // overlap, but the tracker dedups the blocks. + let min_cursor = starts + .iter() + .filter_map(|s| self.active_batches.get(s).map(|b| b.deferred_cursor())) + .min() + .unwrap_or(0); + + // Each batch's filters as a shared `Arc` snapshot, built once and reused + // — never a fresh combined clone per generation (that cost GBs). + let batch_arcs: Vec>> = starts + .iter() + .filter_map(|s| self.active_batches.get_mut(s).map(|b| b.filters_arc())) + .collect(); + + let mut delta: HashMap> = HashMap::new(); + for (wallet_id, s) in &self.deferred[min_cursor..log_len] { + delta.entry(*wallet_id).or_default().insert(s.clone()); } - // Check if batch has pending blocks - if batch.pending_blocks() > 0 { - break; + events.extend(self.match_and_request(batch_arcs, delta).await?); + } + + for &start in &starts { + if let Some(b) = self.active_batches.get_mut(&start) { + b.set_deferred_cursor(log_len); } + } - // Check if rescan is needed and not done - if !batch.rescan_complete() { - // Take per-wallet collected scripts from the batch - let scripts_by_wallet = self - .active_batches - .get_mut(&batch_start) - .map(|b| b.take_collected_scripts()) - .unwrap_or_default(); + crate::timer::P_BULK.add(t_bulk.elapsed()); + Ok(events) + } - if !scripts_by_wallet.is_empty() { - // Rescan current batch - events.extend(self.rescan_batch(batch_start, &scripts_by_wallet).await?); + /// Match `new_scripts` against a combined filter snapshot spanning several + /// open batches, then route each matched block to the batch that owns its + /// height (bumping that batch's `pending_blocks` so its commit waits) and + /// emit one `BlocksNeeded` for the whole wave. + async fn match_and_request( + &mut self, + batch_arcs: Vec>>, + new_scripts: HashMap>, + ) -> SyncResult> { + let mut synced_heights: HashMap = HashMap::new(); + let mut filter_elements: HashMap>> = HashMap::new(); + { + let wallet = self.wallet.read().await; + for id in new_scripts.keys() { + synced_heights.insert(*id, wallet.wallet_synced_height(id)); + filter_elements.insert(*id, wallet.monitored_filter_elements_for(id)); + } + } - // Also rescan later batches that are already scanned - let later_batches: Vec = self - .active_batches - .iter() - .filter(|(&start, batch)| start > batch_start && batch.scanned()) - .map(|(&start, _)| start) - .collect(); + let mut block_to_wallets: BTreeMap> = BTreeMap::new(); + for (wallet_id, scripts) in &new_scripts { + let elements = filter_elements.get(wallet_id).map(Vec::as_slice).unwrap_or(&[]); + if scripts.is_empty() && elements.is_empty() { + continue; + } + let scripts_vec: Vec = scripts.iter().cloned().collect(); + let min_synced = synced_heights.get(wallet_id).copied().unwrap_or(0); + let matches = + match_filters_multi(batch_arcs.clone(), scripts_vec, elements.to_vec(), min_synced) + .await; + for key in matches { + block_to_wallets.entry(key).or_default().insert(*wallet_id); + } + } - for later_start in later_batches { - events.extend(self.rescan_batch(later_start, &scripts_by_wallet).await?); + let mut events = Vec::new(); + let mut blocks_needed: BTreeMap> = BTreeMap::new(); + if !block_to_wallets.is_empty() { + self.progress.add_matched(block_to_wallets.len() as u32); + } + for (key, wallets) in block_to_wallets { + // Map the matched height to the open batch that owns it (BTreeMap + // range: largest start <= height, if height is within its range). + let batch_start = self + .active_batches + .range(..=key.height()) + .next_back() + .filter(|(_, b)| key.height() <= b.end_height()) + .map(|(&s, _)| s); + let Some(batch_start) = batch_start else { + continue; + }; + match self.tracker.track_for_new_scripts(&key, batch_start, wallets) { + BlockTrackResult::NewlyTracked { + wallets, + } => { + blocks_needed.insert(key, wallets); + if let Some(b) = self.active_batches.get_mut(&batch_start) { + b.set_pending_blocks(b.pending_blocks() + 1); } + } + BlockTrackResult::InFlight { + wallets, + } => { + blocks_needed.insert(key, wallets); + } + BlockTrackResult::AlreadyProcessed => {} + } + } + if !blocks_needed.is_empty() { + events.push(SyncEvent::BlocksNeeded { + blocks: blocks_needed, + }); + } + Ok(events) + } + + /// Commit settled batches in order: scanned, no blocks pending, and matched + /// against the ENTIRE derived-script log. + /// + /// The head does that last match HERE rather than waiting for a `bulk_rescan` + /// pass, which only fires when the block pipeline drains — during a download it + /// almost never does, so gating commits on it would freeze the frontier and pile + /// every scanned batch up in memory. `bulk_rescan` still runs the same match + /// ahead of the frontier for the other open batches, so their blocks are already + /// downloading by the time the frontier arrives: download and matching overlap. + async fn try_commit_batches(&mut self) -> SyncResult> { + let t_commit = std::time::Instant::now(); + let mut events = Vec::new(); - // Check if rescan found more blocks - if let Some(batch) = self.active_batches.get(&batch_start) { - if batch.pending_blocks() > 0 { - // Found more blocks, can't commit yet - break; + loop { + let log_len = self.deferred.len(); + let Some((&batch_start, batch)) = self.active_batches.first_key_value() else { + break; + }; + if !batch.scanned() { + break; + } + if batch.pending_blocks() > 0 { + break; + } + if batch.deferred_cursor() < log_len { + let cursor = batch.deferred_cursor(); + { + let t_head = std::time::Instant::now(); + let arc = self.active_batches.get_mut(&batch_start).map(|b| b.filters_arc()); + if let Some(arc) = arc { + let mut delta: HashMap> = HashMap::new(); + for (wallet_id, s) in &self.deferred[cursor..log_len] { + delta.entry(*wallet_id).or_default().insert(s.clone()); } + events.extend(self.match_and_request(vec![arc], delta).await?); } + crate::timer::P_HEAD.add(t_head.elapsed()); } - // Mark rescan as complete - if let Some(batch) = self.active_batches.get_mut(&batch_start) { - batch.mark_rescan_complete(); + if let Some(b) = self.active_batches.get_mut(&batch_start) { + b.set_deferred_cursor(log_len); + } + if self.active_batches.get(&batch_start).is_some_and(|b| b.pending_blocks() > 0) { + break; // that match queued blocks; wait for them } } - // Commit this batch. Advance per-wallet `synced_height` only for - // wallets that were behind for this batch at scan time. Already-synced - // wallets are never touched. let batch = self.active_batches.remove(&batch_start).unwrap(); let end = batch.end_height(); if end > self.progress.committed_height() { @@ -541,30 +1044,32 @@ impl SyncResult> { + let t_scan = std::time::Instant::now(); let mut events = Vec::new(); - // Collect batch starts that need scanning - let batch_starts: Vec = self + let starts: Vec = self .active_batches .iter() .filter(|(_, batch)| { @@ -572,19 +1077,161 @@ impl = Vec::new(); + for wallet_id in &behind { + let scripts = wallet.monitored_script_pubkeys_for(wallet_id); + let elements = wallet.monitored_filter_elements_for(wallet_id); + if !scripts.is_empty() || !elements.is_empty() { + states.push(WalletScanState { + id: *wallet_id, + synced: wallet.wallet_synced_height(wallet_id), + scripts, + elements, + }); + } + } + let mut per_batch: Vec<(u32, BTreeSet)> = Vec::with_capacity(starts.len()); + for &s in &starts { + let end = self.active_batches.get(&s).map(|b| b.end_height()).unwrap_or(0); + per_batch.push((s, wallet.wallets_behind(end))); + } + (states, per_batch) + }; + + let mut arcs: Vec>> = + Vec::with_capacity(starts.len()); + for (s, behind) in behind_by_batch { + if let Some(b) = self.active_batches.get_mut(&s) { + b.mark_scanned(); + b.set_scanned_wallets(behind); + let arc = b.filters_arc(); + if !arc.is_empty() { + arcs.push(arc); + } + } + } + + if arcs.is_empty() || wallet_states.is_empty() { + crate::timer::P_SCAN.add(t_scan.elapsed()); + return Ok(events); + } + + let union_scripts: Vec = + wallet_states.iter().flat_map(|s| s.scripts.iter().cloned()).collect(); + let union_elements: Vec> = + wallet_states.iter().flat_map(|s| s.elements.iter().cloned()).collect(); + let min_synced = wallet_states.iter().map(|s| s.synced).min().unwrap_or(0); + + let wallet_queries: Vec<(WalletId, u32, FilterQuery)> = wallet_states + .iter() + .map(|s| { + let mut query: FilterQuery = s.scripts.iter().map(|sp| sp.as_bytes()).collect(); + for element in &s.elements { + query.push(element); + } + (s.id, s.synced, query) + }) + .collect(); + + let matches = + match_filters_multi(arcs.clone(), union_scripts, union_elements, min_synced).await; + + let filter_by_key: HashMap<&FilterMatchKey, &BlockFilter> = + arcs.iter().flat_map(|a| a.iter().map(|(k, f)| (k, f))).collect(); + let mut block_to_wallets: BTreeMap> = BTreeMap::new(); + for key in matches { + let Some(filter) = filter_by_key.get(&key) else { + continue; + }; + for (wallet_id, wallet_synced, query) in &wallet_queries { + if key.height() <= *wallet_synced { + continue; + } + match filter.match_any(key.hash(), query) { + Ok(true) => { + block_to_wallets.entry(key.clone()).or_default().insert(*wallet_id); + } + Ok(false) => {} + Err(e) => tracing::warn!( + "filter match_any error during attribution at height {}: {}; treating as non-match", + key.height(), + e + ), + } + } + } + + tracing::info!( + "Scanned {} batches in one pass: {} matching blocks across {} wallets", + arcs.len(), + block_to_wallets.len(), + wallet_states.len() + ); + + if !block_to_wallets.is_empty() { + self.progress.add_matched(block_to_wallets.len() as u32); - for batch_start in batch_starts { - events.extend(self.scan_batch(batch_start).await?); + let mut blocks_needed: BTreeMap> = BTreeMap::new(); + for (key, wallets) in block_to_wallets { + let Some(owner) = self + .active_batches + .range(..=key.height()) + .next_back() + .filter(|(_, b)| key.height() <= b.end_height()) + .map(|(&s, _)| s) + else { + continue; + }; + match self.tracker.track(&key, owner, wallets) { + BlockTrackResult::NewlyTracked { + wallets, + } => { + blocks_needed.insert(key, wallets); + if let Some(b) = self.active_batches.get_mut(&owner) { + b.set_pending_blocks(b.pending_blocks() + 1); + } + } + BlockTrackResult::InFlight { + wallets, + } => { + blocks_needed.insert(key, wallets); + } + BlockTrackResult::AlreadyProcessed => {} + } + } + if !blocks_needed.is_empty() { + events.push(SyncEvent::BlocksNeeded { + blocks: blocks_needed, + }); + } } + crate::timer::P_SCAN.add(t_scan.elapsed()); Ok(events) } - /// Create lookahead batches up to MAX_LOOKAHEAD_BATCHES. + /// Create lookahead batches all the way up to the available filter-header + /// tip. Unbounded: filters are requested as far ahead as headers allow. async fn try_create_lookahead_batches(&mut self) -> SyncResult> { let mut events = Vec::new(); - while self.active_batches.len() < MAX_LOOKAHEAD_BATCHES { + loop { // Find where next batch should start let next_start = if let Some((&_, last_batch)) = self.active_batches.last_key_value() { last_batch.end_height() + 1 @@ -630,119 +1277,13 @@ impl>, - ) -> SyncResult> { - if new_scripts.is_empty() { - return Ok(vec![]); - } - - let Some(batch) = self.active_batches.get(&batch_start) else { - return Ok(vec![]); - }; - - tracing::info!( - "Rescan filters ({}-{}) for new scripts across {} wallets", - batch.start_height(), - batch.end_height(), - new_scripts.len() - ); - - if batch.filters().is_empty() { - return Ok(vec![]); - } - let batch_filters = batch.filters(); - - // Per-wallet `synced_height` snapshot so heights below the wallet's - // own progress are skipped during the rescan, plus the bare - // owner/voting filter elements a compact filter carries beyond the - // wallet's scriptPubKeys. - let mut synced_heights: HashMap = HashMap::new(); - let mut filter_elements: HashMap>> = HashMap::new(); - { - let wallet = self.wallet.read().await; - for id in new_scripts.keys() { - synced_heights.insert(*id, wallet.wallet_synced_height(id)); - filter_elements.insert(*id, wallet.monitored_filter_elements_for(id)); - } - } - - let mut block_to_wallets: BTreeMap> = BTreeMap::new(); - for (wallet_id, scripts) in new_scripts { - let elements = filter_elements.get(wallet_id).map(Vec::as_slice).unwrap_or(&[]); - if scripts.is_empty() && elements.is_empty() { - continue; - } - let scripts_vec: Vec = scripts.iter().cloned().collect(); - let min_synced = synced_heights.get(wallet_id).copied().unwrap_or(0); - let matches = check_compact_filters_for_elements( - batch_filters, - &scripts_vec, - elements, - min_synced, - ); - for key in matches { - block_to_wallets.entry(key).or_default().insert(*wallet_id); - } - } - - let mut events = Vec::new(); - let mut blocks_needed: BTreeMap> = BTreeMap::new(); - let mut new_blocks_count = 0; - - if !block_to_wallets.is_empty() { - self.progress.add_matched(block_to_wallets.len() as u32); - } - for (key, wallets) in block_to_wallets { - // Matches here are driven by scripts that did not exist when the - // block was first processed, so a processed record must not - // suppress the re-download: the block has to be re-applied - // against the extended pools. - match self.tracker.track_for_new_scripts(&key, batch_start, wallets) { - BlockTrackResult::NewlyTracked { - wallets, - } => { - blocks_needed.insert(key, wallets); - new_blocks_count += 1; - } - BlockTrackResult::InFlight { - wallets, - } => { - // Block already on its way; merge late wallet ids into the - // pipeline's pending wallet set via a fresh BlocksNeeded. - blocks_needed.insert(key, wallets); - } - // Never returned by track_for_new_scripts. - BlockTrackResult::AlreadyProcessed => {} - } - } - - // Update batch pending_blocks count for the genuinely new entries only. - if new_blocks_count > 0 { - if let Some(batch) = self.active_batches.get_mut(&batch_start) { - batch.set_pending_blocks(batch.pending_blocks() + new_blocks_count); - } - tracing::info!("Rescan found {} additional blocks", new_blocks_count); - } - if !blocks_needed.is_empty() { - events.push(SyncEvent::BlocksNeeded { - blocks: blocks_needed, - }); - } - - Ok(events) - } - - /// Scan a specific batch, matching its filters against each behind-wallet's - /// addresses individually so already-synced wallets are not redundantly - /// rescanned. - async fn scan_batch(&mut self, batch_start: u32) -> SyncResult> { - let mut events = Vec::new(); + /// Scan a specific batch, matching its filters against each behind-wallet's + /// addresses individually so already-synced wallets are not redundantly + /// rescanned. + async fn scan_batch(&mut self, batch_start: u32) -> SyncResult> { + let t_scan = std::time::Instant::now(); + let _g = scopeguard_scan(t_scan); + let mut events = Vec::new(); let (batch_end, filters_empty) = { let Some(batch) = self.active_batches.get_mut(&batch_start) else { @@ -832,51 +1373,58 @@ impl> = { + let Some(batch) = self.active_batches.get_mut(&batch_start) else { return Ok(events); }; - let batch_filters = batch.filters(); + batch.filters_arc() + }; - let matches = check_compact_filters_for_elements( - batch_filters, - &union_scripts, - &union_elements, - min_synced, - ); - let mut block_to_wallets: BTreeMap> = - BTreeMap::new(); - for key in matches { - let Some(filter) = batch_filters.get(&key) else { - tracing::warn!( - "skipping unmatched filter key at height {}: hash {}", - key.height(), - key.hash() - ); + let matches = match_filters_parallel( + batch_filters.clone(), + union_scripts, + union_elements, + min_synced, + ) + .await; + + // Attribute each matched block to the wallets whose query actually hit it. + let filter_by_key: HashMap<&FilterMatchKey, &BlockFilter> = + batch_filters.iter().map(|(k, f)| (k, f)).collect(); + let mut block_to_wallets: BTreeMap> = BTreeMap::new(); + for key in matches { + let Some(filter) = filter_by_key.get(&key) else { + tracing::warn!( + "skipping unmatched filter key at height {}: hash {}", + key.height(), + key.hash() + ); + continue; + }; + for (wallet_id, wallet_synced, query) in &wallet_queries { + if key.height() <= *wallet_synced { continue; - }; - for (wallet_id, wallet_synced, query) in &wallet_queries { - if key.height() <= *wallet_synced { - continue; - } - let matched = match filter.match_any(key.hash(), query) { - Ok(matched) => matched, - Err(e) => { - tracing::warn!( - "filter match_any error during attribution at height {}: {}; treating as non-match", - key.height(), - e - ); - false - } - }; - if matched { - block_to_wallets.entry(key.clone()).or_default().insert(*wallet_id); + } + let matched = match filter.match_any(key.hash(), query) { + Ok(matched) => matched, + Err(e) => { + tracing::warn!( + "filter match_any error during attribution at height {}: {}; treating as non-match", + key.height(), + e + ); + false } + }; + if matched { + block_to_wallets.entry(key.clone()).or_default().insert(*wallet_id); } } - block_to_wallets - }; + } tracing::info!( "Batch {}-{}: found {} matching blocks across {} behind wallets", @@ -930,12 +1478,28 @@ impl u32 { + let block_tip = + self.header_storage.read().await.get_tip().await.map(|t| t.height()).unwrap_or(0); + block_tip.max(self.progress.filter_header_tip_height()) + } + /// Handle notification that new filter headers are available. /// Used by both FilterHeadersSyncComplete and FilterHeadersStored events. pub(super) async fn handle_new_filter_headers( &mut self, tip_height: u32, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { self.progress.update_filter_header_tip_height(tip_height); self.update_target_height(tip_height); @@ -950,10 +1514,15 @@ impl {} } @@ -988,1919 +1557,3 @@ impl; - type MultiTestFiltersManager = FiltersManager< - PersistentBlockHeaderStorage, - PersistentFilterHeaderStorage, - PersistentFilterStorage, - MultiMockWallet, - >; - type TestSyncManager = dyn SyncManager; - - async fn create_test_manager() -> TestFiltersManager { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let wallet = Arc::new(RwLock::new(MockWallet::new())); - FiltersManager::new( - wallet, - storage.block_headers(), - storage.filter_headers(), - storage.filters(), - ) - .await - } - - async fn create_multi_test_manager( - wallet: Arc>, - ) -> MultiTestFiltersManager { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - FiltersManager::new( - wallet, - storage.block_headers(), - storage.filter_headers(), - storage.filters(), - ) - .await - } - - /// Set up a manager fully synced to height 100: block headers stored - /// through the boundary block 101, filter bodies and filter headers - /// persisted through 100, and progress anchored at 100. The returned - /// boundary filter body for height 101 is fed in over the wire by callers. - /// State is left untouched so each caller picks the reconnect or boot path. - async fn setup_synced_manager_at_tip() -> (TestFiltersManager, Vec
, BlockFilter) { - let mut manager = create_test_manager().await; - - let headers = Header::dummy_batch(0..102); - manager - .header_storage - .write() - .await - .store_headers(&headers.iter().map(HashedBlockHeader::from).collect::>()) - .await - .unwrap(); - - let boundary_filter = BlockFilter::new(&[0u8; 32]); - { - let mut fs = manager.filter_storage.write().await; - for h in 0..=100 { - fs.store_filter(h, &boundary_filter.content).await.unwrap(); - } - } - - // The prev header is non-zero because the segment store treats an - // all-zero filter header as an empty (sentinel) slot. - let prev_filter_header = FilterHeader::from_byte_array([1u8; 32]); - manager - .filter_header_storage - .write() - .await - .store_filter_headers_at_height( - &[prev_filter_header, boundary_filter.filter_header(&prev_filter_header)], - 100, - ) - .await - .unwrap(); - - manager.wallet.write().await.update_wallet_synced_height(&MOCK_WALLET_ID, 100); - manager.progress.update_committed_height(100); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_target_height(100); - - (manager, headers, boundary_filter) - } - - /// Build a real `BlockFilter` for a single-output block paying `address`. - fn filter_for_address( - height: u32, - address: &dashcore::Address, - ) -> (FilterMatchKey, BlockFilter) { - let tx = Transaction::dummy(address, 0..0, &[height as u64]); - let block = Block::dummy(height, vec![tx]); - let filter = BlockFilter::dummy(&block); - (FilterMatchKey::new(height, block.block_hash()), filter) - } - - #[tokio::test] - async fn test_filters_manager_new() { - let manager = create_test_manager().await; - assert_eq!(manager.identifier(), ManagerIdentifier::Filter); - assert_eq!(manager.state(), SyncState::WaitForEvents); - assert_eq!(manager.wanted_message_types(), vec![MessageType::CFilter]); - assert_eq!(manager.progress.committed_height(), 0); - assert_eq!(manager.progress.stored_height(), 0); - assert_eq!(manager.progress.target_height(), 0); - assert_eq!(manager.progress.filter_header_tip_height(), 0); - } - - #[tokio::test] - async fn test_filters_manager_new_restores_from_storage() { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - - // Set wallet committed height via last_processed_height (MockWallet default delegates) - let mut wallet = MockWallet::new(); - wallet.update_wallet_synced_height(&MOCK_WALLET_ID, 50); - let wallet = Arc::new(RwLock::new(wallet)); - - // Pre-populate filter storage with filters at heights 1..=100 - let filters = storage.filters(); - { - let mut filter_store = filters.write().await; - for height in 1..=100 { - filter_store.store_filter(height, &[0u8; 32]).await.unwrap(); - } - } - - // Pre-populate block header storage with 300 headers for target_height - let block_headers = Header::dummy_batch(0..300); - storage - .block_headers() - .write() - .await - .store_headers( - &block_headers - .iter() - .map(crate::types::HashedBlockHeader::from) - .collect::>(), - ) - .await - .unwrap(); - - // Pre-populate filter header storage with headers at heights 1..=200 - let filter_headers = storage.filter_headers(); - { - let dummy_headers = vec![FilterHeader::all_zeros(); 200]; - filter_headers - .write() - .await - .store_filter_headers_at_height(&dummy_headers, 1) - .await - .unwrap(); - } - - let manager = FiltersManager::new( - wallet, - storage.block_headers(), - storage.filter_headers(), - storage.filters(), - ) - .await; - - assert_eq!(manager.progress.committed_height(), 50); - assert_eq!(manager.progress.stored_height(), 100); - assert_eq!(manager.progress.target_height(), 299); - assert_eq!(manager.progress.filter_header_tip_height(), 200); - } - - #[tokio::test] - async fn test_filters_manager_progress() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - manager.progress.update_stored_height(500); - manager.progress.update_target_height(1000); - manager.progress.add_processed(350); - manager.progress.add_downloaded(250); - manager.progress.add_matched(150); - - let manager_ref: &TestSyncManager = &manager; - let progress = manager_ref.progress(); - if let SyncManagerProgress::Filters(progress) = progress { - assert_eq!(progress.state(), SyncState::Syncing); - assert_eq!(progress.stored_height(), 500); - assert_eq!(progress.target_height(), 1000); - assert_eq!(progress.processed(), 350); - assert_eq!(progress.downloaded(), 250); - assert_eq!(progress.matched(), 150); - assert!(progress.last_activity().elapsed().as_secs() < 1); - } else { - panic!("Expected SyncManagerProgress::Filters"); - } - } - - #[tokio::test] - async fn test_filter_header_tip_height_is_monotonic() { - let mut manager = create_test_manager().await; - manager.progress.update_filter_header_tip_height(500); - assert_eq!(manager.progress.filter_header_tip_height(), 500); - - // Lower value is rejected - manager.progress.update_filter_header_tip_height(200); - assert_eq!(manager.progress.filter_header_tip_height(), 500); - - // Equal value is rejected (no spurious activity bump) - manager.progress.update_filter_header_tip_height(500); - assert_eq!(manager.progress.filter_header_tip_height(), 500); - - // Higher value is accepted - manager.progress.update_filter_header_tip_height(600); - assert_eq!(manager.progress.filter_header_tip_height(), 600); - } - - #[tokio::test] - async fn test_max_lookahead_constant() { - // Verify the constant is set to expected value - assert_eq!(MAX_LOOKAHEAD_BATCHES, 3); - } - - #[tokio::test] - async fn test_batch_commit_blocks_on_pending() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - - // Manually create two batches - let mut batch1 = FiltersBatch::new(0, 4999, HashMap::new()); - let batch2 = FiltersBatch::new(5000, 9999, HashMap::new()); - - // batch1 has pending blocks, batch2 does not - batch1.set_pending_blocks(1); - - manager.active_batches.insert(0, batch1); - manager.active_batches.insert(5000, batch2); - - // Try to commit - should not commit anything since batch1 has pending blocks - manager.try_commit_batches().await.unwrap(); - assert_eq!(manager.active_batches.len(), 2); - // committed_height stays at initial value since nothing was committed - assert!(manager.active_batches.contains_key(&0)); - } - - #[tokio::test] - async fn test_batch_commit_succeeds_when_ready() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - - // Create a batch with no pending blocks, scanned, and rescan complete - let mut batch1 = FiltersBatch::new(0, 4999, HashMap::new()); - batch1.set_pending_blocks(0); - batch1.mark_scanned(); - batch1.mark_rescan_complete(); - - manager.active_batches.insert(0, batch1); - - // Commit should work - manager.try_commit_batches().await.unwrap(); - assert_eq!(manager.active_batches.len(), 0); - assert_eq!(manager.progress.committed_height(), 4999); - // No wallets were recorded as scanned for this batch, so the per-wallet - // synced_height stays at its initial value. - assert_eq!(manager.wallet.read().await.wallet_synced_height(&MOCK_WALLET_ID), 0); - } - - #[tokio::test] - async fn test_batch_commit_advances_only_scanned_wallets() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - - // First batch records MOCK_WALLET_ID as scanned, so its synced_height - // advances to the batch end on commit. - let mut batch1 = FiltersBatch::new(0, 4999, HashMap::new()); - batch1.set_pending_blocks(0); - batch1.mark_scanned(); - batch1.mark_rescan_complete(); - batch1.set_scanned_wallets(BTreeSet::from([MOCK_WALLET_ID])); - manager.active_batches.insert(0, batch1); - - manager.try_commit_batches().await.unwrap(); - assert_eq!(manager.progress.committed_height(), 4999); - assert_eq!(manager.wallet.read().await.wallet_synced_height(&MOCK_WALLET_ID), 4999); - - // Second batch leaves scanned_wallets empty (nothing to scan in this - // range), so the per-wallet synced_height stays put even though the - // committed_height advances. - let mut batch2 = FiltersBatch::new(5000, 9999, HashMap::new()); - batch2.set_pending_blocks(0); - batch2.mark_scanned(); - batch2.mark_rescan_complete(); - manager.active_batches.insert(5000, batch2); - - manager.try_commit_batches().await.unwrap(); - assert_eq!(manager.progress.committed_height(), 9999); - assert_eq!(manager.wallet.read().await.wallet_synced_height(&MOCK_WALLET_ID), 4999); - } - - /// Two wallets in the same batch: only the wallet recorded in - /// `scanned_wallets` advances, the other stays put even after commit. - #[tokio::test] - async fn test_batch_commit_advances_only_recorded_wallet_with_two_wallets() { - let wallet_a: WalletId = [0xAA; 32]; - let wallet_b: WalletId = [0xBB; 32]; - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet(wallet_a, MockWalletState::default()); - w.insert_wallet(wallet_b, MockWalletState::default()); - } - let mut manager = create_multi_test_manager(multi.clone()).await; - manager.set_state(SyncState::Syncing); - - // Batch records only wallet_a as scanned. wallet_b is excluded. - let mut batch = FiltersBatch::new(0, 4999, HashMap::new()); - batch.set_pending_blocks(0); - batch.mark_scanned(); - batch.mark_rescan_complete(); - batch.set_scanned_wallets(BTreeSet::from([wallet_a])); - manager.active_batches.insert(0, batch); - - manager.try_commit_batches().await.unwrap(); - assert_eq!(manager.progress.committed_height(), 4999); - assert_eq!(multi.read().await.wallet_synced_height(&wallet_a), 4999); - assert_eq!(multi.read().await.wallet_synced_height(&wallet_b), 0); - } - - /// `scan_batch` with two wallets at different `synced_height` values: - /// only the wallet whose synced_height is below the matching block's - /// height should be attributed. - #[tokio::test] - async fn test_scan_batch_attributes_per_wallet_height() { - let wallet_low: WalletId = [0x01; 32]; - let wallet_high: WalletId = [0x02; 32]; - let address_low = dashcore::Address::dummy(Network::Regtest, 1); - let address_high = dashcore::Address::dummy(Network::Regtest, 2); - - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - // wallet_low is behind: synced_height=10, will see filters above 10. - w.insert_wallet( - wallet_low, - MockWalletState { - addresses: vec![address_low.clone()], - synced_height: 10, - last_processed_height: 10, - }, - ); - // wallet_high is mostly synced: synced_height=50, only sees > 50. - w.insert_wallet( - wallet_high, - MockWalletState { - addresses: vec![address_high.clone()], - synced_height: 50, - last_processed_height: 50, - }, - ); - } - let mut manager = create_multi_test_manager(multi).await; - manager.set_state(SyncState::Syncing); - - // Build a batch with three filters: at 30 paying wallet_low's address, - // at 60 paying wallet_high's address, at 70 paying wallet_low's address. - let mut filters: HashMap = HashMap::new(); - let (key_30, f_30) = filter_for_address(30, &address_low); - let (key_60, f_60) = filter_for_address(60, &address_high); - let (key_70, f_70) = filter_for_address(70, &address_low); - filters.insert(key_30.clone(), f_30); - filters.insert(key_60.clone(), f_60); - filters.insert(key_70.clone(), f_70); - - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - manager.progress.update_stored_height(99); - - let events = manager.scan_batch(0).await.unwrap(); - - // Find the BlocksNeeded event. - let blocks = events - .iter() - .find_map(|e| match e { - SyncEvent::BlocksNeeded { - blocks, - } => Some(blocks), - _ => None, - }) - .expect("BlocksNeeded event"); - - // Block at 30 only attributable to wallet_low (height <= wallet_high.synced) - let attr_30 = blocks.get(&key_30).expect("entry for height 30"); - assert!(attr_30.contains(&wallet_low)); - assert!(!attr_30.contains(&wallet_high)); - - // Block at 60 only attributable to wallet_high (matches its address); - // wallet_low's address does not match so it shouldn't be there either. - let attr_60 = blocks.get(&key_60).expect("entry for height 60"); - assert!(attr_60.contains(&wallet_high)); - assert!(!attr_60.contains(&wallet_low)); - - // Block at 70 only attributable to wallet_low: matches wallet_low's - // address, and wallet_high's address does not match this filter. - let attr_70 = blocks.get(&key_70).expect("entry for height 70"); - assert!(attr_70.contains(&wallet_low)); - assert!(!attr_70.contains(&wallet_high)); - } - - /// `rescan_batch` with multiple wallets in `scripts_by_wallet`: - /// each wallet's new scripts are matched independently and the - /// attribution is correct in the emitted `BlocksNeeded`. - #[tokio::test] - async fn test_rescan_batch_attributes_per_wallet_addresses() { - let wallet_a: WalletId = [0x0A; 32]; - let wallet_b: WalletId = [0x0B; 32]; - let address_a = dashcore::Address::dummy(Network::Regtest, 11); - let address_b = dashcore::Address::dummy(Network::Regtest, 22); - - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet(wallet_a, MockWalletState::default()); - w.insert_wallet(wallet_b, MockWalletState::default()); - } - let mut manager = create_multi_test_manager(multi).await; - manager.set_state(SyncState::Syncing); - - let mut filters: HashMap = HashMap::new(); - let (key_a, f_a) = filter_for_address(15, &address_a); - let (key_b, f_b) = filter_for_address(25, &address_b); - filters.insert(key_a.clone(), f_a); - filters.insert(key_b.clone(), f_b); - - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - - let mut new_scripts: HashMap> = HashMap::new(); - new_scripts.insert(wallet_a, HashSet::from([address_a.script_pubkey()])); - new_scripts.insert(wallet_b, HashSet::from([address_b.script_pubkey()])); - - let events = manager.rescan_batch(0, &new_scripts).await.unwrap(); - - let blocks = events - .iter() - .find_map(|e| match e { - SyncEvent::BlocksNeeded { - blocks, - } => Some(blocks), - _ => None, - }) - .expect("BlocksNeeded event"); - - let attr_a = blocks.get(&key_a).expect("entry for wallet_a's match"); - assert!(attr_a.contains(&wallet_a)); - assert!(!attr_a.contains(&wallet_b)); - - let attr_b = blocks.get(&key_b).expect("entry for wallet_b's match"); - assert!(attr_b.contains(&wallet_b)); - assert!(!attr_b.contains(&wallet_a)); - } - - /// A block that was already processed for a wallet is re-queued by - /// `rescan_batch` when newly derived scripts match it: the prior - /// processing predates those scripts, so the block must be re-applied - /// against the extended pools. Plain scans keep skipping processed - /// blocks. - #[tokio::test] - async fn test_rescan_batch_reprocesses_already_processed_block() { - let address = Address::dummy(Network::Regtest, 33); - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - - let (key, filter) = filter_for_address(20, &address); - let mut filters: HashMap = HashMap::new(); - filters.insert(key.clone(), filter); - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - - // The block was downloaded and processed for the wallet before the - // rescan script existed. - manager.tracker.record_processed(20, *key.hash(), &BTreeSet::from([MOCK_WALLET_ID])); - - let mut new_scripts: HashMap> = HashMap::new(); - new_scripts.insert(MOCK_WALLET_ID, HashSet::from([address.script_pubkey()])); - - let events = manager.rescan_batch(0, &new_scripts).await.unwrap(); - - let blocks = events - .iter() - .find_map(|e| match e { - SyncEvent::BlocksNeeded { - blocks, - } => Some(blocks), - _ => None, - }) - .expect("BlocksNeeded event for the re-queued block"); - assert!(blocks.get(&key).expect("re-queued block entry").contains(&MOCK_WALLET_ID)); - - // The re-queued block is accounted exactly once so the batch cannot - // commit before its re-processing completes. - assert_eq!(manager.active_batches.get(&0).unwrap().pending_blocks(), 1); - } - - /// `rescan_batch` honours each wallet's own `synced_height`: a new - /// address belonging to a wallet that has already advanced past a height - /// must not produce a `BlocksNeeded` for that height, even when the - /// filter for that height matches the new address. Two wallets at - /// different heights are exercised so that both the include-above and - /// skip-below paths run. - #[tokio::test] - async fn test_rescan_batch_skips_below_per_wallet_synced_height() { - let wallet_low: WalletId = [0xA1; 32]; - let wallet_high: WalletId = [0xA2; 32]; - let address_low = dashcore::Address::dummy(Network::Regtest, 41); - let address_high = dashcore::Address::dummy(Network::Regtest, 42); - - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet( - wallet_low, - MockWalletState { - addresses: vec![], - synced_height: 20, - last_processed_height: 20, - }, - ); - w.insert_wallet( - wallet_high, - MockWalletState { - addresses: vec![], - synced_height: 60, - last_processed_height: 60, - }, - ); - } - let mut manager = create_multi_test_manager(multi).await; - manager.set_state(SyncState::Syncing); - - // Filters at 30 (matches wallet_low) and 70 (matches wallet_high). - // For wallet_low (synced=20), height 30 is fresh and 70 is also fresh - // since 70 > 20. For wallet_high (synced=60), height 30 is below its - // synced_height so it must be skipped, while 70 is fresh. - let (key_30, f_30) = filter_for_address(30, &address_low); - let (key_70, f_70) = filter_for_address(70, &address_high); - let mut filters: HashMap = HashMap::new(); - filters.insert(key_30.clone(), f_30); - filters.insert(key_70.clone(), f_70); - - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - - // wallet_high also "discovers" address_low's script to demonstrate - // that even when a new script would match a low height, the per-wallet - // synced_height filter prevents emitting it. - let mut new_scripts: HashMap> = HashMap::new(); - new_scripts.insert(wallet_low, HashSet::from([address_low.script_pubkey()])); - new_scripts.insert( - wallet_high, - HashSet::from([address_low.script_pubkey(), address_high.script_pubkey()]), - ); - - let events = manager.rescan_batch(0, &new_scripts).await.unwrap(); - - let blocks = events - .iter() - .find_map(|e| match e { - SyncEvent::BlocksNeeded { - blocks, - } => Some(blocks), - _ => None, - }) - .expect("BlocksNeeded event"); - - // wallet_low must see height 30, wallet_high must NOT (synced=60>30). - let attr_30 = blocks.get(&key_30).expect("entry at height 30 for wallet_low"); - assert!(attr_30.contains(&wallet_low)); - assert!(!attr_30.contains(&wallet_high)); - - // wallet_high must see height 70 since 70 > 60. - let attr_70 = blocks.get(&key_70).expect("entry at height 70 for wallet_high"); - assert!(attr_70.contains(&wallet_high)); - } - - /// `scan_batch` for a behind wallet with no monitored addresses still - /// records the wallet in `scanned_wallets` so its `synced_height` - /// advances at commit. Otherwise zero-address wallets would be listed by - /// `wallets_behind` on every batch forever. - #[tokio::test] - async fn test_scan_batch_advances_zero_address_wallet() { - let wallet_id: WalletId = [0xCC; 32]; - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet(wallet_id, MockWalletState::default()); - } - let mut manager = create_multi_test_manager(multi.clone()).await; - manager.set_state(SyncState::Syncing); - - // Batch with one filter at height 50 (irrelevant: wallet has no addresses). - let mut filters: HashMap = HashMap::new(); - let throwaway_address = dashcore::Address::dummy(Network::Regtest, 99); - let (key, filter) = filter_for_address(50, &throwaway_address); - filters.insert(key, filter); - - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - manager.progress.update_stored_height(99); - - let events = manager.scan_batch(0).await.unwrap(); - assert!(events.is_empty(), "no addresses should mean no BlocksNeeded events"); - - // Mark batch ready so commit can run, then commit. - if let Some(b) = manager.active_batches.get_mut(&0) { - b.set_pending_blocks(0); - b.mark_rescan_complete(); - } - manager.try_commit_batches().await.unwrap(); - - // Wallet had no addresses, but it was behind, so its synced_height - // advances to the batch end after commit. - assert_eq!(multi.read().await.wallet_synced_height(&wallet_id), 99); - } - - /// `scan_batch` after a runtime-added wallet whose address matches a - /// block already in flight must re-emit `BlocksNeeded` so the - /// `BlocksPipeline` merges the new wallet id into the pending set. - #[tokio::test] - async fn test_scan_batch_in_flight_re_emits_for_late_wallet() { - let wallet_id: WalletId = [0xDD; 32]; - let address = dashcore::Address::dummy(Network::Regtest, 7); - - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet( - wallet_id, - MockWalletState { - addresses: vec![address.clone()], - synced_height: 0, - last_processed_height: 0, - }, - ); - } - let mut manager = create_multi_test_manager(multi).await; - manager.set_state(SyncState::Syncing); - - // One matching filter at height 40. - let (key_40, f_40) = filter_for_address(40, &address); - let mut filters: HashMap = HashMap::new(); - filters.insert(key_40.clone(), f_40); - - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - manager.progress.update_stored_height(99); - - // Pre-seed the tracker so `tracker.track` returns InFlight. - manager.tracker.track(&key_40, 0, BTreeSet::from([wallet_id])); - - let events = manager.scan_batch(0).await.unwrap(); - - let blocks = events - .iter() - .find_map(|e| match e { - SyncEvent::BlocksNeeded { - blocks, - } => Some(blocks), - _ => None, - }) - .expect("InFlight path must still emit BlocksNeeded for wallet-set merge"); - let attribution = blocks.get(&key_40).expect("entry for the in-flight block"); - assert!(attribution.contains(&wallet_id)); - } - - /// `scan_batch` `AlreadyProcessed` path: when every candidate wallet has - /// already had this block processed, the block is skipped (no - /// `BlocksNeeded`). - #[tokio::test] - async fn test_scan_batch_already_processed_is_skipped() { - let wallet_id: WalletId = [0xEE; 32]; - let address = dashcore::Address::dummy(Network::Regtest, 8); - - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet( - wallet_id, - MockWalletState { - addresses: vec![address.clone()], - synced_height: 0, - last_processed_height: 0, - }, - ); - } - let mut manager = create_multi_test_manager(multi).await; - manager.set_state(SyncState::Syncing); - - let (key_40, f_40) = filter_for_address(40, &address); - let mut filters: HashMap = HashMap::new(); - filters.insert(key_40.clone(), f_40); - - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - manager.progress.update_stored_height(99); - - // Pre-record processing for the only candidate wallet so the residual - // is empty and `tracker.track` returns `AlreadyProcessed`. - manager.tracker.record_processed(40, *key_40.hash(), &BTreeSet::from([wallet_id])); - - let events = manager.scan_batch(0).await.unwrap(); - let has_blocks_needed = events.iter().any(|e| matches!(e, SyncEvent::BlocksNeeded { .. })); - assert!(!has_blocks_needed, "AlreadyProcessed must not emit BlocksNeeded"); - } - - /// `scan_batch` for a wallet added at runtime whose address matches a - /// block already processed for another wallet must re-emit `BlocksNeeded` - /// with only the late wallet in the attribution set so the block reloads - /// from storage and applies for the late wallet without disturbing the - /// already-processed one. - #[tokio::test] - async fn test_scan_batch_late_wallet_recovers_already_processed_block() { - let early: WalletId = [0xE1; 32]; - let late: WalletId = [0xE2; 32]; - let address = dashcore::Address::dummy(Network::Regtest, 9); - - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet( - early, - MockWalletState { - addresses: vec![address.clone()], - synced_height: 0, - last_processed_height: 0, - }, - ); - w.insert_wallet( - late, - MockWalletState { - addresses: vec![address.clone()], - synced_height: 0, - last_processed_height: 0, - }, - ); - } - let mut manager = create_multi_test_manager(multi).await; - manager.set_state(SyncState::Syncing); - - let (key_40, f_40) = filter_for_address(40, &address); - let mut filters: HashMap = HashMap::new(); - filters.insert(key_40.clone(), f_40); - - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - manager.progress.update_stored_height(99); - - // The early wallet has already had this block applied. The late - // wallet has not. Both wallets' addresses match the filter at 40. - manager.tracker.record_processed(40, *key_40.hash(), &BTreeSet::from([early])); - - let events = manager.scan_batch(0).await.unwrap(); - let blocks = events - .iter() - .find_map(|e| match e { - SyncEvent::BlocksNeeded { - blocks, - } => Some(blocks), - _ => None, - }) - .expect("late wallet must trigger a BlocksNeeded re-emit"); - let attribution = blocks.get(&key_40).expect("entry for the recovered block"); - assert!(attribution.contains(&late), "late wallet must receive the block"); - assert!( - !attribution.contains(&early), - "early wallet was already processed for this block, must be excluded" - ); - } - - /// `try_commit_batches` prunes `processed_blocks_per_wallet` entries at - /// or below the new committed_height, since they cannot be reached again - /// without `reset_for_rescan` wiping the map outright. - #[tokio::test] - async fn test_commit_prunes_processed_blocks_per_wallet() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - - let wallet_id: WalletId = [0xFA; 32]; - let hash_in = dashcore::block::Header::dummy(0).block_hash(); - let hash_out = dashcore::block::Header::dummy(1).block_hash(); - let key_in = FilterMatchKey::new(2500, hash_in); - let key_out = FilterMatchKey::new(7500, hash_out); - manager.tracker.record_processed(2500, hash_in, &BTreeSet::from([wallet_id])); - manager.tracker.record_processed(7500, hash_out, &BTreeSet::from([wallet_id])); - - // Batch 0..=4999 is ready to commit; pruning drops the 2500 entry but - // keeps the 7500 entry which sits above the new committed_height. - let mut batch = FiltersBatch::new(0, 4999, HashMap::new()); - batch.set_pending_blocks(0); - batch.mark_scanned(); - batch.mark_rescan_complete(); - manager.active_batches.insert(0, batch); - - manager.try_commit_batches().await.unwrap(); - - assert_eq!(manager.progress.committed_height(), 4999); - // The 2500 record is gone: a fresh `track` for the same wallet - // re-tracks the block instead of returning `AlreadyProcessed`. - assert!(matches!( - manager.tracker.track(&key_in, 0, BTreeSet::from([wallet_id])), - BlockTrackResult::NewlyTracked { .. } - )); - // The 7500 record survives above the committed height. - assert_eq!( - manager.tracker.track(&key_out, 0, BTreeSet::from([wallet_id])), - BlockTrackResult::AlreadyProcessed - ); - } - - /// `tick` rescan with a wallet that has a non-zero `synced_height`: the - /// batch must start at `synced_height + 1`, not at genesis. - #[tokio::test] - async fn test_tick_rescans_from_wallet_synced_height_not_genesis() { - let mut manager = create_test_manager().await; - - // Wallet sits at synced_height=150, manager committed at 300, so - // the wallet falls behind and the rescan trigger fires. - manager.wallet.write().await.update_wallet_synced_height(&MOCK_WALLET_ID, 150); - manager.set_state(SyncState::Synced); - manager.progress.update_committed_height(300); - manager.progress.update_stored_height(300); - manager.progress.update_filter_header_tip_height(300); - manager.progress.update_target_height(300); - - // Headers must exist in storage so start_download can resolve them. - let headers = dashcore::block::Header::dummy_batch(0..301); - manager - .header_storage - .write() - .await - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let (tx, _rx) = unbounded_channel(); - let _ = manager.tick(&RequestSender::new(tx)).await.unwrap(); - - // Batch must start at 151, not at 0. - assert!(manager.active_batches.contains_key(&151)); - assert!(!manager.active_batches.contains_key(&0)); - } - - /// scan_batch's union-then-attribute pass must not falsely attribute a - /// block to a wallet whose own address does not actually match the - /// filter, even if the union pass picked up the block. - #[tokio::test] - async fn test_scan_batch_attribution_excludes_non_matching_wallet() { - let wallet_a: WalletId = [0xAA; 32]; - let wallet_b: WalletId = [0xBB; 32]; - let address_a = dashcore::Address::dummy(Network::Regtest, 31); - let address_b = dashcore::Address::dummy(Network::Regtest, 32); - - let multi = MultiMockWallet::new(); - let multi = Arc::new(RwLock::new(multi)); - { - let mut w = multi.write().await; - w.insert_wallet( - wallet_a, - MockWalletState { - addresses: vec![address_a.clone()], - synced_height: 0, - last_processed_height: 0, - }, - ); - w.insert_wallet( - wallet_b, - MockWalletState { - addresses: vec![address_b.clone()], - synced_height: 0, - last_processed_height: 0, - }, - ); - } - let mut manager = create_multi_test_manager(multi).await; - manager.set_state(SyncState::Syncing); - - // Filter at height 40 only matches address_a. address_b is in the - // union but does not match this specific filter, so the attribution - // pass must exclude wallet_b. - let (key_40, f_40) = filter_for_address(40, &address_a); - let mut filters: HashMap = HashMap::new(); - filters.insert(key_40.clone(), f_40); - - let mut batch = FiltersBatch::new(0, 99, filters); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - manager.progress.update_stored_height(99); - - let events = manager.scan_batch(0).await.unwrap(); - let blocks = events - .iter() - .find_map(|e| match e { - SyncEvent::BlocksNeeded { - blocks, - } => Some(blocks), - _ => None, - }) - .expect("BlocksNeeded event"); - let attribution = blocks.get(&key_40).expect("entry for the matching block"); - assert!(attribution.contains(&wallet_a)); - assert!(!attribution.contains(&wallet_b)); - } - - #[tokio::test] - async fn test_batch_commit_order_preserved() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - - // Create two batches, both ready to commit - let mut batch1 = FiltersBatch::new(0, 4999, HashMap::new()); - batch1.set_pending_blocks(0); - batch1.mark_scanned(); - batch1.mark_rescan_complete(); - - let mut batch2 = FiltersBatch::new(5000, 9999, HashMap::new()); - batch2.set_pending_blocks(0); - batch2.mark_scanned(); - batch2.mark_rescan_complete(); - - manager.active_batches.insert(5000, batch2); // Insert higher one first - manager.active_batches.insert(0, batch1); - - // Commit should commit both in order - manager.try_commit_batches().await.unwrap(); - assert_eq!(manager.active_batches.len(), 0); - assert_eq!(manager.progress.committed_height(), 9999); // Both committed - } - - #[tokio::test] - async fn test_blocks_remaining_tracks_batch() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::Syncing); - - let wallet: WalletId = [1; 32]; - let hash1 = dashcore::block::Header::dummy(0).block_hash(); - let hash2 = dashcore::block::Header::dummy(1).block_hash(); - - // Track blocks from two different batches. - manager.tracker.track(&FilterMatchKey::new(100, hash1), 0, BTreeSet::from([wallet])); - manager.tracker.track(&FilterMatchKey::new(5100, hash2), 5000, BTreeSet::from([wallet])); - - // Each block round-trips its (height, batch_start) on `finish_in_flight`. - assert_eq!(manager.tracker.finish_in_flight(&hash1), Some((100, 0))); - assert_eq!(manager.tracker.finish_in_flight(&hash2), Some((5100, 5000))); - } - - #[tokio::test] - async fn test_track_block_match_per_wallet_residual() { - let mut manager = create_test_manager().await; - let hash = dashcore::block::Header::dummy(0).block_hash(); - let key = FilterMatchKey::new(100, hash); - let wallet_a: WalletId = [0xA1; 32]; - let wallet_b: WalletId = [0xB2; 32]; - - // First match for {A}: nothing tracked yet, helper records the block. - assert_eq!( - manager.tracker.track(&key, 0, BTreeSet::from([wallet_a])), - BlockTrackResult::NewlyTracked { - wallets: BTreeSet::from([wallet_a]) - } - ); - - // Second match for {A} while still in flight: residual is {A} (no - // processing has been recorded yet), so InFlight re-emits to merge - // late-arriving wallet ids into the pipeline's pending set. - assert_eq!( - manager.tracker.track(&key, 0, BTreeSet::from([wallet_a])), - BlockTrackResult::InFlight { - wallets: BTreeSet::from([wallet_a]) - } - ); - - // Block is delivered and processed for {A}. Round-trip the (height, - // batch_start) tuple while removing the in-flight entry, then record - // the processing. - assert_eq!(manager.tracker.finish_in_flight(&hash), Some((100, 0))); - manager.tracker.record_processed(100, hash, &BTreeSet::from([wallet_a])); - - // Late-added wallet B's filter matches the same block. A is already - // processed, B is not — residual is {B} and it gets re-queued via - // NewlyTracked so the block reloads from storage and applies for B - // only. - assert_eq!( - manager.tracker.track(&key, 5000, BTreeSet::from([wallet_a, wallet_b])), - BlockTrackResult::NewlyTracked { - wallets: BTreeSet::from([wallet_b]) - } - ); - assert_eq!(manager.tracker.finish_in_flight(&hash), Some((100, 5000))); - - // After B is also processed, a third match including only A and B - // returns AlreadyProcessed since both are covered. - manager.tracker.record_processed(100, hash, &BTreeSet::from([wallet_b])); - assert_eq!( - manager.tracker.track(&key, 5000, BTreeSet::from([wallet_a, wallet_b])), - BlockTrackResult::AlreadyProcessed - ); - assert!(manager.tracker.finish_in_flight(&hash).is_none()); - } - - #[tokio::test] - async fn test_is_idle() { - let mut manager = create_test_manager().await; - let hash = dashcore::block::Header::dummy(0).block_hash(); - let key = FilterMatchKey::new(100, hash); - let wallet_id: WalletId = [0xCC; 32]; - - // Fresh manager is idle - assert!(manager.is_idle()); - - // Test each involved field separately - manager.active_batches.insert(0, FiltersBatch::new(0, 999, HashMap::new())); - assert!(!manager.is_idle()); - manager.active_batches.clear(); - - manager.tracker.track(&key, 0, BTreeSet::from([wallet_id])); - assert!(!manager.is_idle()); - manager.tracker.clear(); - - manager.tracker.record_processed(100, hash, &BTreeSet::from([wallet_id])); - assert!(!manager.is_idle()); - manager.tracker.clear(); - - manager.pending_batches.insert(FiltersBatch::new(0, 999, HashMap::new())); - assert!(!manager.is_idle()); - manager.pending_batches.clear(); - - manager.filter_pipeline.init(0, 999); - assert!(!manager.is_idle()); - manager.filter_pipeline = FiltersPipeline::new(); - - // Populate all fields, then reset_for_rescan restores idleness - manager.active_batches.insert(0, FiltersBatch::new(0, 999, HashMap::new())); - manager.tracker.track(&key, 0, BTreeSet::from([wallet_id])); - manager.tracker.record_processed(100, hash, &BTreeSet::from([wallet_id])); - manager.pending_batches.insert(FiltersBatch::new(1000, 1999, HashMap::new())); - manager.filter_pipeline.init(2000, 2999); - assert!(!manager.is_idle()); - - manager.reset_for_rescan(); - assert!(manager.is_idle()); - } - - #[tokio::test] - async fn test_batch_collects_scripts() { - use crate::sync::filters::batch::FiltersBatch; - use dashcore::Network; - - let mut batch = FiltersBatch::new(0, 4999, HashMap::new()); - - // Initially empty - assert!(batch.take_collected_scripts().is_empty()); - - // Add scripts using test utility - let script1 = dashcore::Address::dummy(Network::Testnet, 1).script_pubkey(); - let script2 = dashcore::Address::dummy(Network::Testnet, 2).script_pubkey(); - let wallet_id: WalletId = [7; 32]; - - batch.add_scripts_for_wallet(wallet_id, [script1.clone(), script2.clone()]); - - let collected = batch.take_collected_scripts(); - let for_wallet = collected.get(&wallet_id).expect("wallet entry"); - assert_eq!(for_wallet.len(), 2); - assert!(for_wallet.contains(&script1)); - assert!(for_wallet.contains(&script2)); - - // After take, should be empty - assert!(batch.take_collected_scripts().is_empty()); - } - - #[tokio::test] - async fn test_start_download_waits_when_filter_headers_insufficient() { - let mut manager = create_test_manager().await; - assert_eq!(manager.state(), SyncState::WaitForEvents); - - // Wallet committed to height 100, so scan_start will be 101 - manager.wallet.write().await.update_wallet_synced_height(&MOCK_WALLET_ID, 100); - // Filter headers only reached 50, so its below scan_start - manager.progress.update_filter_header_tip_height(50); - // Chain tip higher so the Synced early-return is not taken - manager.progress.update_target_height(1000); - - let (tx, _rx) = unbounded_channel(); - let events = manager.start_download(&RequestSender::new(tx)).await.unwrap(); - - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::WaitForEvents); - assert!(manager.is_idle()); - } - - #[tokio::test] - async fn test_start_download_transitions_to_syncing_when_filters_available() { - let mut manager = create_test_manager().await; - assert_eq!(manager.state(), SyncState::WaitForEvents); - - // Store headers so send_pending can resolve stop hashes - let headers = dashcore::block::Header::dummy_batch(0..101); - manager - .header_storage - .write() - .await - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - // Filter headers available up to 100, wallet at genesis (scan_start = 0) - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_target_height(1000); - - let (tx, _rx) = unbounded_channel(); - let events = manager.start_download(&RequestSender::new(tx)).await.unwrap(); - - assert_eq!(manager.state(), SyncState::Syncing); - assert!(!manager.is_idle()); - assert!(events.is_empty()); - // Should have created an initial processing batch spanning scan_start to filter tip - let batch = manager.active_batches.get(&0).expect("batch at scan_start=0"); - assert_eq!(batch.start_height(), 0); - assert_eq!(batch.end_height(), 100); - } - - #[tokio::test] - async fn test_handle_new_filter_headers_transitions_synced_to_syncing() { - let mut manager = create_test_manager().await; - - // Simulate fully synced state at height 100 - manager.set_state(SyncState::Synced); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_committed_height(100); - manager.progress.update_target_height(1000); - // Pipeline target at 150 with no pending batches, so extend_target(150) - // is a no-op and send_pending returns immediately (no headers needed) - manager.filter_pipeline.init(151, 150); - // The active batch ends at 200, so try_create_lookahead_batches chains - // its next start off 201, past the filter header tip 150, and the bound - // stops it before creating any batch or emitting an event. - manager.active_batches.insert(101, FiltersBatch::new(101, 200, HashMap::new())); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - // New filter headers arrive at 150: committed(100) < tip(150) - let events = manager.handle_new_filter_headers(150, &requests).await.unwrap(); - - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::Syncing); - assert!(!manager.is_idle()); - } - - #[tokio::test] - async fn test_handle_new_filter_headers_synced_restart() { - let mut manager = create_test_manager().await; - - // Store block headers so start_download can resolve heights - let headers = dashcore::block::Header::dummy_batch(0..101); - manager - .header_storage - .write() - .await - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - // Simulate restart where everything is already synced but state is WaitForEvents. - // committed == stored == filter_header_tip — start_download detects synced state. - manager.set_state(SyncState::WaitForEvents); - manager.wallet.write().await.update_wallet_synced_height(&MOCK_WALLET_ID, 100); - manager.progress.update_committed_height(100); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_target_height(100); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - let events = manager.handle_new_filter_headers(100, &requests).await.unwrap(); - - assert_eq!(manager.state(), SyncState::Synced); - assert!( - events.iter().any(|e| matches!( - e, - SyncEvent::FiltersSyncComplete { - tip_height: 100 - } - )), - "expected FiltersSyncComplete(100), got {:?}", - events - ); - assert!(manager.active_batches.is_empty()); - } - - /// A node that boots already synced takes the `start_download` early - /// return. That path must still record the scan frontier in - /// `processing_height`, otherwise the next block creates a lookahead batch - /// from a stale 0 and reads filters at height 0 that were never stored - /// (panicking on the empty leading segment). - #[tokio::test] - async fn test_handle_new_filter_headers_new_block_starts_at_frontier() { - let mut manager = create_test_manager().await; - - // Headers cover the synced tip plus the new block. Filter storage is - // intentionally left empty: with the old behavior the lookahead batch - // reads from height 0 and trips the empty-segment guard. - let headers = dashcore::block::Header::dummy_batch(0..102); - manager - .header_storage - .write() - .await - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - manager.set_state(SyncState::WaitForEvents); - manager.wallet.write().await.update_wallet_synced_height(&MOCK_WALLET_ID, 100); - manager.progress.update_committed_height(100); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_target_height(100); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - // Boot already synced: start_download detects the synced state and - // returns early, but must still advance the scan frontier. - let events = manager.start_download(&requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Synced); - assert!(events.iter().any(|e| matches!( - e, - SyncEvent::FiltersSyncComplete { - tip_height: 100 - } - ))); - assert_eq!(manager.processing_height, 101); - - // A new block extends the chain; the lookahead batch must start at the - // frontier, not at height 0. - manager.handle_new_filter_headers(101, &requests).await.unwrap(); - assert!(!manager.active_batches.contains_key(&0)); - assert_eq!(manager.active_batches.keys().next(), Some(&101)); - } - - /// A fully synced node that reconnects and then sees one new block must - /// commit it. `start_sync` takes the `stored == committed == tip` branch, - /// reports `Synced`, and anchors the store and processing cursors at the - /// frontier; the boundary block's filter header, body, and commit then flow - /// through to the new tip. If those cursors were left at 0, the boundary - /// block would create a batch from height 0 and wedge the in-order store - /// loop so `committed_height` freezes one below tip. - #[tokio::test] - async fn test_start_sync_synced_restart_then_boundary_block_commits() { - let (mut manager, headers, boundary_filter) = setup_synced_manager_at_tip().await; - - // Fully-synced restart: `start_sync` requires `WaitingForConnections`. - manager.set_state(SyncState::WaitingForConnections); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - // Reconnect: the already-synced branch reports completion and anchors the - // store/processing cursors at the frontier. - let events = manager.start_sync(&requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Synced); - assert!(events.iter().any(|e| matches!( - e, - SyncEvent::FiltersSyncComplete { - tip_height: 100 - } - ))); - - // Block 101's filter header lands: the manager queues its body download - // and a processing batch starting at the frontier (not at height 0). - manager - .handle_sync_event( - &SyncEvent::FilterHeadersStored { - start_height: 101, - end_height: 101, - tip_height: 101, - }, - &requests, - ) - .await - .unwrap(); - - // The peer answers with the boundary filter body over the real path. - let peer: SocketAddr = "127.0.0.1:19999".parse().unwrap(); - let cfilter = CFilter { - filter_type: 0, - block_hash: headers[101].block_hash(), - filter: boundary_filter.content.clone(), - }; - manager - .handle_message(Message::new(peer, NetworkMessage::CFilter(cfilter)), &requests) - .await - .unwrap(); - - // A trailing tick drives any residual processing to completion. - manager.tick(&requests).await.unwrap(); - - assert_eq!(manager.progress.committed_height(), 101); - assert_eq!(manager.state(), SyncState::Synced); - assert!(manager.active_batches.is_empty()); - assert!(manager.next_batch_to_store > 100); - } - - /// A node that boots already synced (default `WaitForEvents` state, which - /// `start_sync` never runs from) must commit a new block that lands after - /// the initial `FilterHeadersSyncComplete`. That event drives - /// `start_download`'s early return, which must anchor the store cursor and - /// park the pipeline at the frontier. Left at their defaults, the next - /// block makes `extend_target` re-request every filter from height 1 and - /// wedges the in-order store loop on a `next_batch_to_store` of 0, freezing - /// `committed_height` one below tip. - #[tokio::test] - async fn test_synced_boot_then_boundary_block_commits() { - let (mut manager, headers, boundary_filter) = setup_synced_manager_at_tip().await; - - // Fully-synced boot leaves the manager in its default state. - assert_eq!(manager.state(), SyncState::WaitForEvents); - - let (tx, mut rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - // Filter header sync completing at the stored tip reports Synced via - // `start_download`'s early return, anchoring the frontier cursors. - let events = manager - .handle_sync_event( - &SyncEvent::FilterHeadersSyncComplete { - tip_height: 100, - }, - &requests, - ) - .await - .unwrap(); - assert_eq!(manager.state(), SyncState::Synced); - assert!(events.iter().any(|e| matches!( - e, - SyncEvent::FiltersSyncComplete { - tip_height: 100 - } - ))); - assert_eq!(manager.next_batch_to_store, 101); - - // Block 101's filter header lands. - manager - .handle_sync_event( - &SyncEvent::FilterHeadersStored { - start_height: 101, - end_height: 101, - tip_height: 101, - }, - &requests, - ) - .await - .unwrap(); - - // Only the boundary body may be requested: a pipeline left unparked - // would re-request every filter from height 1 here. - while let Ok(request) = rx.try_recv() { - let (NetworkRequest::SendMessage(msg) - | NetworkRequest::SendMessageToPeer(msg, _) - | NetworkRequest::BroadcastMessage(msg)) = request; - if let NetworkMessage::GetCFilters(get) = msg { - assert_eq!(get.start_height, 101, "unexpected filter re-download"); - } - } - - // The peer answers with the boundary filter body over the real path. - let peer: SocketAddr = "127.0.0.1:19999".parse().unwrap(); - let cfilter = CFilter { - filter_type: 0, - block_hash: headers[101].block_hash(), - filter: boundary_filter.content.clone(), - }; - manager - .handle_message(Message::new(peer, NetworkMessage::CFilter(cfilter)), &requests) - .await - .unwrap(); - - // A trailing tick drives any residual processing to completion. - manager.tick(&requests).await.unwrap(); - - assert_eq!(manager.progress.committed_height(), 101); - assert_eq!(manager.state(), SyncState::Synced); - assert!(manager.active_batches.is_empty()); - } - - /// A node that restores its filter header tip ahead of its stored filter - /// bodies (headers at 101, bodies and commit at 100) must download and - /// commit the boundary body on reconnect. `start_sync` delegates to the - /// download path, which sees the scan frontier at or below the restored - /// tip and requests the boundary body rather than parking in - /// `WaitForEvents` and never asking for it. - #[tokio::test] - async fn test_start_sync_filter_headers_ahead_of_bodies_commits_boundary() { - let (mut manager, headers, boundary_filter) = setup_synced_manager_at_tip().await; - - // Filter headers restored ahead of the stored filter bodies. - manager.progress.update_filter_header_tip_height(101); - manager.set_state(SyncState::WaitingForConnections); - - let (tx, mut rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - // Reconnect delegates to the download path, which must request the - // boundary body at 101 rather than parking without asking for it. - let events = manager.start_sync(&requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Syncing); - assert!(!events.iter().any(|e| matches!(e, SyncEvent::SyncStart { .. }))); - assert!(manager.active_batches.contains_key(&101)); - - let mut requested_boundary = false; - while let Ok(request) = rx.try_recv() { - let (NetworkRequest::SendMessage(msg) - | NetworkRequest::SendMessageToPeer(msg, _) - | NetworkRequest::BroadcastMessage(msg)) = request; - if let NetworkMessage::GetCFilters(get) = msg { - assert_eq!(get.start_height, 101, "unexpected filter re-download"); - requested_boundary = true; - } - } - assert!(requested_boundary, "boundary filter body 101 must be requested"); - - // The peer answers with the boundary filter body over the real path. - let peer: SocketAddr = "127.0.0.1:19999".parse().unwrap(); - let cfilter = CFilter { - filter_type: 0, - block_hash: headers[101].block_hash(), - filter: boundary_filter.content.clone(), - }; - manager - .handle_message(Message::new(peer, NetworkMessage::CFilter(cfilter)), &requests) - .await - .unwrap(); - - manager.tick(&requests).await.unwrap(); - - assert_eq!(manager.progress.committed_height(), 101); - assert_eq!(manager.state(), SyncState::Synced); - assert!(manager.active_batches.is_empty()); - } - - #[tokio::test] - async fn test_handle_new_filter_headers_stays_synced_when_already_synced() { - let mut manager = create_test_manager().await; - - // Already in Synced state with matching heights — should stay Synced without - // emitting duplicate events. - manager.set_state(SyncState::Synced); - manager.progress.update_committed_height(100); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_target_height(100); - manager.filter_pipeline.init(101, 100); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - let events = manager.handle_new_filter_headers(100, &requests).await.unwrap(); - - assert_eq!(manager.state(), SyncState::Synced); - assert!(events.is_empty()); - } - - /// A wallet whose `synced_height` sits below the manager's `committed_height` - /// must trigger a rescan from the wallet's height. This simulates a wallet - /// being added at runtime behind current scan progress. - #[tokio::test] - async fn test_tick_rescans_when_wallet_falls_behind_committed() { - let mut manager = create_test_manager().await; - - // Set up a single address on the wallet and a real matching filter at - // height 50 so scan_batch can emit a `BlocksNeeded` for it on rescan. - let address = dashcore::Address::dummy(Network::Regtest, 7); - manager.wallet.write().await.set_addresses(vec![address.clone()]); - - // Build matching block + filter at height 50. - let tx = Transaction::dummy(&address, 0..0, &[50u64]); - let block_at_50 = Block::dummy(50, vec![tx]); - let filter_at_50 = BlockFilter::dummy(&block_at_50); - - // Headers must form a contiguous range so the storage segment is - // fully populated. Only the height-50 entry needs to be the real - // header; the rest are dummies and never get matched against. - let mut headers: Vec = dashcore::block::Header::dummy_batch(0..201); - headers[50] = block_at_50.header; - manager - .header_storage - .write() - .await - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - // Persist a filter at every height in 0..=100 so `load_filters` over - // the initial batch range succeeds. Non-matching heights get a - // throwaway filter, only height 50 gets the address-matching one. - let mut filter_store = manager.filter_storage.write().await; - let dummy_filter = BlockFilter::new(&[0u8; 32]); - for h in 0..=100u32 { - if h == 50 { - filter_store.store_filter(h, &filter_at_50.content).await.unwrap(); - } else { - filter_store.store_filter(h, &dummy_filter.content).await.unwrap(); - } - } - drop(filter_store); - - // Manager believes filters are committed up to 100. Filter headers - // and target are pinned at 100 too so start_download immediately - // scans the freshly created batch instead of waiting for downloads. - manager.set_state(SyncState::Synced); - manager.progress.update_committed_height(100); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_target_height(100); - - // Pre-populate in-flight state so we can verify reset_for_rescan runs. - manager.active_batches.insert(101, FiltersBatch::new(101, 200, HashMap::new())); - let stale_hash = dashcore::block::Header::dummy(0).block_hash(); - let stale_key = FilterMatchKey::new(150, stale_hash); - manager.tracker.record_processed(150, stale_hash, &BTreeSet::from([MOCK_WALLET_ID])); - manager.filter_pipeline.init(101, 200); - - // MockWallet defaults to synced_height=0, so wallets_behind(100) = {MOCK_WALLET_ID}. - assert_eq!(manager.wallet.read().await.synced_height(), 0); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - // Sanity: the pre-populated stale processed record is present, so - // `track` for the same wallet would short-circuit to AlreadyProcessed. - assert_eq!( - manager.tracker.track(&stale_key, 0, BTreeSet::from([MOCK_WALLET_ID])), - BlockTrackResult::AlreadyProcessed - ); - // Undo the side effect of the probing `track` so the original - // processed record is the only state present going into `tick`. - manager.tracker.clear(); - manager.tracker.record_processed(150, stale_hash, &BTreeSet::from([MOCK_WALLET_ID])); - - let events = manager.tick(&requests).await.unwrap(); - - // Old in-flight state was cleared and a fresh batch was created at scan_start=0. - assert!(!manager.active_batches.contains_key(&101)); - assert!(manager.active_batches.contains_key(&0)); - // The stale pre-populated record was wiped by `reset_for_rescan`: - // a fresh `track` for the same wallet now returns `NewlyTracked`. - assert!(matches!( - manager.tracker.track(&stale_key, 0, BTreeSet::from([MOCK_WALLET_ID])), - BlockTrackResult::NewlyTracked { .. } - )); - - // start_download set committed_height to scan_start - 1 = 0. - assert_eq!(manager.progress.committed_height(), 0); - assert_eq!(manager.state(), SyncState::Syncing); - - // Verify a `BlocksNeeded` event was emitted that includes MOCK_WALLET_ID - // for the matching block at height 50. - let blocks_needed = events - .iter() - .find_map(|e| match e { - SyncEvent::BlocksNeeded { - blocks, - } => Some(blocks), - _ => None, - }) - .expect("BlocksNeeded event from rescan"); - let key_50 = FilterMatchKey::new(50, block_at_50.block_hash()); - let attribution = blocks_needed.get(&key_50).expect("entry for matching block 50"); - assert!(attribution.contains(&MOCK_WALLET_ID)); - } - - /// When every managed wallet is at or beyond `committed_height`, the rescan - /// trigger must not fire even though the aggregate `synced_height` could - /// otherwise look stale. - #[tokio::test] - async fn test_tick_does_not_rescan_when_no_wallets_behind() { - let mut manager = create_test_manager().await; - - // Wallet at synced_height=200, manager committed at 100 → no wallets behind. - manager.wallet.write().await.update_wallet_synced_height(&MOCK_WALLET_ID, 200); - - manager.set_state(SyncState::Synced); - manager.progress.update_committed_height(100); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(200); - manager.progress.update_target_height(200); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - let events = manager.tick(&requests).await.unwrap(); - - assert!(events.is_empty()); - assert_eq!(manager.progress.committed_height(), 100); - assert_eq!(manager.state(), SyncState::Synced); - assert!(manager.active_batches.is_empty()); - } - - /// `committed_height = 0` on a fresh manager must not falsely trip the - /// rescan trigger. `wallets_behind(0)` returns an empty set since heights - /// are unsigned, so no wallet can be strictly less than 0. - #[tokio::test] - async fn test_tick_does_not_rescan_at_genesis_committed() { - let mut manager = create_test_manager().await; - // Default state: committed_height=0, wallet synced_height=0, state=WaitForEvents. - assert_eq!(manager.progress.committed_height(), 0); - assert_eq!(manager.state(), SyncState::WaitForEvents); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - let events = manager.tick(&requests).await.unwrap(); - - assert!(events.is_empty()); - assert!(manager.is_idle()); - assert_eq!(manager.state(), SyncState::WaitForEvents); - } - - /// The rescan trigger only fires in `Syncing | Synced | WaitForEvents`. - /// `WaitingForConnections` must be skipped since we're not actively syncing. - #[tokio::test] - async fn test_tick_does_not_rescan_in_waiting_for_connections() { - let mut manager = create_test_manager().await; - manager.set_state(SyncState::WaitingForConnections); - manager.progress.update_committed_height(100); - - // Wallet behind committed — would normally trip the trigger. - assert!(!manager.wallet.read().await.wallets_behind(100).is_empty()); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - let events = manager.tick(&requests).await.unwrap(); - - assert!(events.is_empty()); - // committed_height not lowered, no batches created. - assert_eq!(manager.progress.committed_height(), 100); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - assert!(manager.active_batches.is_empty()); - } - - /// `on_disconnect` for `FiltersManager` keeps the block-match tracker, - /// active batches (with their `pending_blocks` counters), pending verified - /// batches, and per-batch filter receipts, and moves any in-flight - /// `getcfilters` slots back to pending so the next `send_pending` reissues - /// them. This keeps `FiltersManager.tracker` in lockstep with - /// `BlocksManager`'s preserved pipeline so already-counted matches do not - /// leak into a permanent non-zero `pending_blocks`. - #[tokio::test] - async fn test_on_disconnect_preserves_in_progress_work() { - let mut manager = create_test_manager().await; - - let key = FilterMatchKey::new(100, dashcore::block::Header::dummy(0).block_hash()); - let wallet_id = MOCK_WALLET_ID; - - let mut batch = FiltersBatch::new(0, 999, HashMap::new()); - batch.set_pending_blocks(1); - manager.active_batches.insert(0, batch); - manager.tracker.track(&key, 0, BTreeSet::from([wallet_id])); - manager.pending_batches.insert(FiltersBatch::new(1000, 1999, HashMap::new())); - manager.filter_pipeline.init(0, 1999); - - manager.on_disconnect(); - - assert!(manager.active_batches.contains_key(&0)); - assert_eq!( - manager.active_batches.get(&0).unwrap().pending_blocks(), - 1, - "active batch's pending_blocks counter must not be reset" - ); - assert_eq!( - manager.tracker.track(&key, 0, BTreeSet::from([wallet_id])), - BlockTrackResult::InFlight { - wallets: BTreeSet::from([wallet_id]) - }, - "tracker must still see the hash as in-flight" - ); - assert_eq!(manager.pending_batches.len(), 1); - - // Sanity check: the full-reset path (`reset_for_rescan`) does wipe - // everything — the two paths must remain distinct. - manager.reset_for_rescan(); - assert!(manager.active_batches.is_empty()); - assert!(manager.pending_batches.is_empty()); - assert!(manager.tracker.is_empty()); - } - - #[tokio::test] - async fn test_handle_new_filter_headers_starts_scan_when_stored_equals_tip() { - let mut manager = create_test_manager().await; - - let headers = dashcore::block::Header::dummy_batch(0..101); - manager - .header_storage - .write() - .await - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - // Populate filter storage so load_filters succeeds during lookahead - { - let dummy_filter = BlockFilter::new(&[0u8; 32]); - let mut fs = manager.filter_storage.write().await; - for h in 0..=100 { - fs.store_filter(h, &dummy_filter.content).await.unwrap(); - } - } - - // Filters stored but not yet committed (restart scenario). - manager.set_state(SyncState::Synced); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_stored_height(100); - manager.progress.update_committed_height(0); - manager.progress.update_target_height(1000); - manager.filter_pipeline.init(101, 100); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - // committed(0) < tip(100) fires the guard even though stored == tip. - // Because stored >= tip, send_pending is skipped (no downloads needed). - let _events = manager.handle_new_filter_headers(100, &requests).await.unwrap(); - - assert_eq!(manager.state(), SyncState::Syncing); - assert!( - manager.active_batches.contains_key(&0), - "expected lookahead batch at processing_height=0, got keys: {:?}", - manager.active_batches.keys().collect::>() - ); - } - - /// When the filter header tip grows while a historical rescan is still - /// draining `active_batches`, the boundary block must still get a - /// processing batch. `handle_new_filter_headers` must create the lookahead - /// batch even with a non-empty active set, or a block landing during a - /// restart reinit window gets its body downloaded but never a processing - /// batch, freezing `committed_height` one below tip. Asserted directly - /// after the call (no `tick`) so the deterministic path is verified in - /// isolation. - #[tokio::test] - async fn test_handle_new_filter_headers_extends_boundary_during_rescan() { - let mut manager = create_test_manager().await; - - let headers = Header::dummy_batch(0..102); - manager - .header_storage - .write() - .await - .store_headers(&headers.iter().map(HashedBlockHeader::from).collect::>()) - .await - .unwrap(); - { - let dummy_filter = BlockFilter::new(&[0u8; 32]); - let mut fs = manager.filter_storage.write().await; - for h in 0..=101 { - fs.store_filter(h, &dummy_filter.content).await.unwrap(); - } - } - - // Historical rescan in progress: an active batch ends at the old tip - // 100, filters stored through 101, committed still climbing. - manager.set_state(SyncState::Syncing); - manager.progress.update_committed_height(0); - manager.progress.update_stored_height(101); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_target_height(101); - manager.active_batches.insert(0, FiltersBatch::new(0, 100, HashMap::new())); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - // New block 101 arrives: tip grows past the in-flight rescan boundary. - manager.handle_new_filter_headers(101, &requests).await.unwrap(); - - assert!( - manager.active_batches.contains_key(&101), - "boundary block 101 must get a processing batch even while a rescan \ - is draining active_batches, got keys: {:?}", - manager.active_batches.keys().collect::>() - ); - } - - #[tokio::test] - async fn test_handle_new_filter_headers_skips_when_fully_committed() { - let mut manager = create_test_manager().await; - - // Everything committed, stored, and at tip. No work to do. - manager.set_state(SyncState::Synced); - manager.progress.update_committed_height(100); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_target_height(1000); - manager.filter_pipeline.init(101, 100); - - let (tx, _rx) = unbounded_channel(); - let requests = RequestSender::new(tx); - - let events = manager.handle_new_filter_headers(100, &requests).await.unwrap(); - - assert_eq!(manager.state(), SyncState::Synced); - assert!(events.is_empty()); - assert!(manager.active_batches.is_empty()); - } - - #[tokio::test] - async fn test_try_process_batch_commits_after_scan_in_same_call() { - let mut manager = create_test_manager().await; - - let headers = dashcore::block::Header::dummy_batch(0..101); - manager - .header_storage - .write() - .await - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - { - let dummy_filter = BlockFilter::new(&[0u8; 32]); - let mut fs = manager.filter_storage.write().await; - for h in 0..=100 { - fs.store_filter(h, &dummy_filter.content).await.unwrap(); - } - } - - manager.set_state(SyncState::Syncing); - manager.progress.update_stored_height(100); - manager.progress.update_filter_header_tip_height(100); - manager.progress.update_committed_height(0); - manager.progress.update_target_height(100); - manager.processing_height = 0; - - // Create a batch that has all filters stored but is not yet scanned. - // Phase 1 (commit) skips it because it's not scanned. - // Phase 2 (scan) marks it scanned. - // Phase 3 (second commit) commits it immediately. - let mut batch = FiltersBatch::new(0, 100, manager.load_filters(0, 100).await.unwrap()); - batch.mark_verified(); - manager.active_batches.insert(0, batch); - - let events = manager.try_process_batch().await.unwrap(); - - // Batch was scanned and committed in a single try_process_batch call. - assert!(manager.active_batches.is_empty()); - assert_eq!(manager.progress.committed_height(), 100); - assert!( - events.iter().any(|e| matches!( - e, - SyncEvent::FiltersSyncComplete { - tip_height: 100 - } - )), - "expected FiltersSyncComplete, got {:?}", - events - ); - } -} diff --git a/dash-spv/src/sync/filters/pipeline.rs b/dash-spv/src/sync/filters/pipeline.rs index acfbd3906..014cd8f5f 100644 --- a/dash-spv/src/sync/filters/pipeline.rs +++ b/dash-spv/src/sync/filters/pipeline.rs @@ -1,45 +1,39 @@ //! CFilters pipeline implementation. //! //! Handles pipelined download of compact block filters (BIP 157/158). -//! Uses DownloadCoordinator for batch-level tracking, with additional +//! Tracks batch start heights with a DownloadCoordinator, plus //! per-batch tracking for individual filter responses. //! //! Filters are buffered in a HashMap until the entire batch //! is complete, enabling batch verification and direct wallet matching. use std::collections::{BTreeSet, HashMap}; -use std::time::Duration; +use std::sync::Arc; +use dashcore::network::message::NetworkMessage; +use dashcore::network::message_filter::GetCFilters; use dashcore::BlockHash; use crate::error::{SyncError, SyncResult}; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::storage::BlockHeaderStorage; -use crate::sync::download_coordinator::{DownloadConfig, DownloadCoordinator}; +use crate::sync::download_coordinator::DownloadCoordinator; use crate::sync::filters::batch::FiltersBatch; use crate::sync::filters::batch_tracker::BatchTracker; /// Batch size for filter requests. const FILTER_BATCH_SIZE: u32 = 1000; -/// Maximum concurrent filter batch requests. -const MAX_CONCURRENT_FILTER_BATCHES: usize = 20; - -/// Timeout for filter batch requests. -/// Each batch requires 1000 individual filter messages, so allow plenty of time. -const FILTER_TIMEOUT: Duration = Duration::from_secs(30); - /// Pipeline for downloading compact block filters. /// -/// Uses DownloadCoordinator for batch-level download mechanics, -/// with BatchTracker for tracking individual filters within -/// each batch. +/// Tracks batch start heights with a DownloadCoordinator, with +/// BatchTracker for tracking individual filters within each batch. /// /// Filters are buffered until the entire batch is complete, then returned /// via `take_completed_batches()` for verification and matching. #[derive(Debug)] pub(super) struct FiltersPipeline { - /// Core coordinator tracks batch start heights. + /// Coordinates pending/in-flight batch start heights. coordinator: DownloadCoordinator, /// Tracks individual filter receipts per batch (start_height -> tracker). batch_trackers: HashMap, @@ -63,11 +57,7 @@ impl FiltersPipeline { /// Create a new CFilters pipeline. pub(super) fn new() -> Self { Self { - coordinator: DownloadCoordinator::new( - DownloadConfig::default() - .with_max_concurrent(MAX_CONCURRENT_FILTER_BATCHES) - .with_timeout(FILTER_TIMEOUT), - ), + coordinator: DownloadCoordinator::new(), batch_trackers: HashMap::new(), completed_batches: BTreeSet::new(), target_height: 0, @@ -78,7 +68,7 @@ impl FiltersPipeline { /// Returns true if the pipeline has no in-flight or pending work. pub(super) fn is_idle(&self) -> bool { - self.coordinator.active_count() == 0 && self.coordinator.pending_count() == 0 + self.coordinator.is_empty() } /// Take completed batches with their buffered filter data for processing. @@ -88,7 +78,7 @@ impl FiltersPipeline { /// Initialize the pipeline for a sync range. /// - /// Pre-queues all batches for the range using the coordinator's pending queue. + /// Pre-queues all batches for the range in the pending queue. pub(super) fn init(&mut self, start_height: u32, target_height: u32) { self.coordinator.clear(); self.batch_trackers.clear(); @@ -128,10 +118,43 @@ impl FiltersPipeline { } } + /// Re-queue GetCFilters batches whose responses never fully arrived (peer + /// died or filters lost), so the next `send_pending` re-issues them. + pub(super) fn handle_timeouts(&mut self, timeout: std::time::Duration) { + let timed_out = self.coordinator.take_timed_out(timeout); + if !timed_out.is_empty() { + tracing::warn!("Filters: re-queuing {} timed-out batch(es)", timed_out.len()); + self.coordinator.enqueue(timed_out); + } + } + + /// TEMPORARY: filters sitting in incomplete batches. A batch is only consumed once + /// ALL of its ~1000 filters have arrived, so partially-filled trackers hold filters + /// that nothing downstream can touch yet. + pub(super) fn buffered_in_trackers(&self) -> (usize, usize, usize) { + let mut batches = 0; + let mut filters = 0; + let mut bytes = 0; + for t in self.batch_trackers.values() { + let (n, b) = t.buffered(); + if n > 0 { + batches += 1; + filters += n; + bytes += b; + } + } + (batches, filters, bytes) + } + + /// Start the response timeout for a batch the router just put on the wire. + pub(super) fn mark_on_flight(&mut self, start_height: u32) { + self.coordinator.mark_on_flight(&[start_height]); + } + /// Send pending filter requests up to the concurrency limit. pub(super) async fn send_pending( &mut self, - requests: &RequestSender, + network: &Arc, storage: &impl BlockHeaderStorage, ) -> SyncResult { let count = self.coordinator.available_to_send(); @@ -178,7 +201,13 @@ impl FiltersPipeline { } }; - requests.request_filters(start_height, stop_hash)?; + network + .send(NetworkMessage::GetCFilters(GetCFilters { + filter_type: 0u8, + start_height, + stop_hash, + })) + .await; self.coordinator.mark_sent(&[start_height]); @@ -206,10 +235,14 @@ impl FiltersPipeline { filter_data: &[u8], ) -> Option { // Find which batch this filter belongs to + let tf = std::time::Instant::now(); let batch_start = self.find_batch_for_height(height)?; + crate::timer::P_FIND_BATCH.add(tf.elapsed()); let tracker = self.batch_trackers.get_mut(&batch_start)?; + let ti = std::time::Instant::now(); tracker.insert_filter(height, block_hash, filter_data); + crate::timer::P_INSERT_FILTER.add(ti.elapsed()); self.filters_received += 1; self.highest_received = self.highest_received.max(height); @@ -235,6 +268,9 @@ impl FiltersPipeline { self.batch_trackers.get_mut(&batch_start).map(|t| t.take_filters()).unwrap_or_default(); self.batch_trackers.remove(&batch_start); + // If the batch wasn't in-flight, it may still be sitting in the pending + // queue (e.g. completed via a late response); drop it so the next + // `send_pending` doesn't resurrect a finished batch. if !self.coordinator.receive(&batch_start) { self.coordinator.cancel_pending(&batch_start); } @@ -260,920 +296,4 @@ impl FiltersPipeline { } None } - - /// Check for timed out batches and handle retries. - /// - /// Does not remove batch trackers — keeps them to receive any late-arriving filters. - pub(super) fn handle_timeouts(&mut self) { - for start in self.coordinator.check_timeouts() { - self.coordinator.enqueue_retry(start); - } - } - - /// Move in-flight `getcfilters` requests back to pending after a peer - /// disconnect so the next `send_pending` reissues them to the new peer. - /// Per-batch trackers and any partially-received filters within them are - /// preserved — `BatchTracker::insert_filter` is idempotent, so duplicates - /// from the new peer are harmless. - pub(super) fn requeue_in_flight(&mut self) { - self.coordinator.requeue_in_flight(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::network::{NetworkRequest, RequestSender}; - use crate::storage::{PersistentBlockHeaderStorage, PersistentStorage}; - use dashcore::bip158::BlockFilter; - use dashcore::block::Header; - use dashcore::network::message::NetworkMessage; - use dashcore_hashes::Hash; - use key_wallet_manager::FilterMatchKey; - use std::time::Duration; - use tempfile::TempDir; - use tokio::sync::mpsc::unbounded_channel; - // ========================================================================= - // Helper functions - // ========================================================================= - - /// Create a pipeline with short timeout for testing timeouts. - fn create_pipeline_with_short_timeout() -> FiltersPipeline { - FiltersPipeline { - coordinator: DownloadCoordinator::new( - DownloadConfig::default().with_timeout(Duration::from_millis(1)), - ), - batch_trackers: HashMap::new(), - completed_batches: BTreeSet::new(), - target_height: 0, - filters_received: 0, - highest_received: 0, - } - } - - /// Create a pipeline with max_concurrent=2 for testing deferred sends. - fn create_pipeline_with_low_concurrency() -> FiltersPipeline { - FiltersPipeline { - coordinator: DownloadCoordinator::new( - DownloadConfig::default().with_max_concurrent(2).with_timeout(FILTER_TIMEOUT), - ), - batch_trackers: HashMap::new(), - completed_batches: BTreeSet::new(), - target_height: 0, - filters_received: 0, - highest_received: 0, - } - } - - /// Create a test request sender with its receiver. - fn create_test_request_sender( - ) -> (RequestSender, tokio::sync::mpsc::UnboundedReceiver) { - let (tx, rx) = unbounded_channel(); - (RequestSender::new(tx), rx) - } - - /// Generate dummy filter data for testing. - fn dummy_filter_data(height: u32) -> Vec { - vec![height as u8, (height >> 8) as u8, 0x01, 0x02] - } - - // ========================================================================= - // FiltersPipeline Construction Tests - // ========================================================================= - - #[test] - fn test_pipeline_new() { - let pipeline = FiltersPipeline::new(); - - assert_eq!(pipeline.coordinator.active_count(), 0); - assert!(pipeline.batch_trackers.is_empty()); - assert!(pipeline.completed_batches.is_empty()); - assert_eq!(pipeline.target_height, 0); - assert_eq!(pipeline.filters_received, 0); - assert_eq!(pipeline.highest_received, 0); - } - - #[test] - fn test_is_idle() { - let mut pipeline = FiltersPipeline::new(); - assert!(pipeline.is_idle()); - - pipeline.init(0, 999); - assert!(!pipeline.is_idle()); - } - - #[test] - fn test_pipeline_default_trait() { - let default_pipeline = FiltersPipeline::default(); - let new_pipeline = FiltersPipeline::new(); - - assert_eq!( - default_pipeline.coordinator.active_count(), - new_pipeline.coordinator.active_count() - ); - assert_eq!(default_pipeline.target_height, new_pipeline.target_height); - } - - #[test] - fn test_pipeline_init() { - let mut pipeline = FiltersPipeline::new(); - - pipeline.init(100, 500); - - // Should have 1 batch queued (100-500 is 401 filters, fits in 1 batch) - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert_eq!(pipeline.target_height, 500); - assert_eq!(pipeline.highest_received, 99); - assert_eq!(pipeline.filters_received, 0); - } - - #[test] - fn test_pipeline_init_resets_state() { - let mut pipeline = FiltersPipeline::new(); - - // Add some state - pipeline.batch_trackers.insert(0, BatchTracker::new(99)); - pipeline.completed_batches.insert(FiltersBatch::new(100, 199, HashMap::new())); - pipeline.coordinator.mark_sent(&[0]); - pipeline.filters_received = 50; - - // Init should clear old state and set up new batches - pipeline.init(200, 300); - - assert!(pipeline.completed_batches.is_empty()); - assert_eq!(pipeline.coordinator.active_count(), 0); - assert_eq!(pipeline.filters_received, 0); - // 1 batch queued for heights 200-300 - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert_eq!(pipeline.batch_trackers.len(), 1); - assert_eq!(pipeline.batch_trackers.get(&200).unwrap().end_height(), 300); - assert_eq!(pipeline.target_height, 300); - } - - // ========================================================================= - // Target Extension Tests - // ========================================================================= - - #[test] - fn test_extend_target_increases() { - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 100); - - pipeline.extend_target(200); - - assert_eq!(pipeline.target_height, 200); - } - - #[tokio::test] - async fn test_extend_target_contiguous_batches() { - // init's last batch is truncated (3000-3500), extend_target fills from 3501. - // Verify all batches are contiguous after sending. - let headers = Header::dummy_batch(0..6000); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 3500); - pipeline.extend_target(5000); - - let (sender, _rx) = create_test_request_sender(); - pipeline.send_pending(&sender, &storage).await.unwrap(); - - let mut ranges: Vec<(u32, u32)> = pipeline - .batch_trackers - .iter() - .map(|(&start, tracker)| (start, tracker.end_height())) - .collect(); - ranges.sort_by_key(|&(start, _)| start); - - // Verify contiguous: 0-999, 1000-1999, 2000-2999, 3000-3500, 3501-4500, 4501-5000 - for window in ranges.windows(2) { - assert_eq!( - window[0].1 + 1, - window[1].0, - "gap or overlap between batches: {}-{} and {}-{}", - window[0].0, - window[0].1, - window[1].0, - window[1].1 - ); - } - assert_eq!(ranges[3], (3000, 3500)); - assert_eq!(ranges[4], (3501, 4500)); - } - - #[test] - fn test_extend_target_ignores_lower() { - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 100); - - pipeline.extend_target(50); - - assert_eq!(pipeline.target_height, 100); - - pipeline.extend_target(100); - - assert_eq!(pipeline.target_height, 100); - } - - // ========================================================================= - // Receive Tests - // ========================================================================= - - #[test] - fn test_requeue_in_flight_preserves_partial_batch_receipts() { - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 99; - - // One batch in-flight (start_height 0). Receive a filter so the - // tracker has partial state. - pipeline.batch_trackers.insert(0, BatchTracker::new(99)); - pipeline.coordinator.mark_sent(&[0]); - let hash = Header::dummy(50).block_hash(); - pipeline.receive_with_data(50, hash, &dummy_filter_data(50)); - assert_eq!(pipeline.filters_received, 1); - assert_eq!(pipeline.coordinator.active_count(), 1); - - pipeline.requeue_in_flight(); - - // Batch is back in pending; tracker (and the partial filter inside it) - // is preserved so the new peer's response merges idempotently. - assert_eq!(pipeline.coordinator.active_count(), 0); - assert_eq!(pipeline.coordinator.pending_count(), 1); - let tracker = pipeline.batch_trackers.get(&0).expect("tracker preserved"); - assert_eq!(tracker.received(), 1); - assert_eq!(pipeline.filters_received, 1); - assert_eq!(pipeline.highest_received, 50); - } - - #[test] - fn test_late_filter_after_requeue_completes_batch_without_orphaning_pending() { - // Regression: a late `cfilter` from the disconnected peer can complete - // a batch after `requeue_in_flight` moved it back to pending. Without - // the cancel-pending hook, the key would linger in `pending` while the - // tracker was gone, and the next `send_pending` would error with - // `SyncError::InvalidState`. - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 2; - - pipeline.batch_trackers.insert(0, BatchTracker::new(2)); - pipeline.coordinator.mark_sent(&[0]); - - // Two filters arrive before disconnect. - for h in 0..=1 { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - pipeline.requeue_in_flight(); - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert_eq!(pipeline.coordinator.active_count(), 0); - - // Late buffered filter from old peer completes the batch. - let hash = Header::dummy(2).block_hash(); - pipeline.receive_with_data(2, hash, &dummy_filter_data(2)); - - assert_eq!(pipeline.completed_batches.len(), 1); - assert!(pipeline.batch_trackers.is_empty()); - // The orphaned pending key must be gone so `send_pending` does not - // resurrect a finished batch. - assert_eq!(pipeline.coordinator.pending_count(), 0); - assert_eq!(pipeline.coordinator.active_count(), 0); - } - - #[test] - fn test_receive_single_filter() { - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 99; - - // Set up batch tracker manually (simulating an in-flight batch) - pipeline.batch_trackers.insert(0, BatchTracker::new(99)); - pipeline.coordinator.mark_sent(&[0]); - - let height = 50; - let hash = Header::dummy(height).block_hash(); - let result = pipeline.receive_with_data(height, hash, &dummy_filter_data(height)); - - // Returns None since batch is not complete (only 1 of 100 filters received) - assert_eq!(result, None); - // But counters are updated - assert_eq!(pipeline.filters_received, 1); - assert_eq!(pipeline.highest_received, 50); - } - - #[test] - fn test_receive_unknown_height() { - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 99; - - // No batch tracker set up - filter is unexpected - let hash = Header::dummy(50).block_hash(); - let result = pipeline.receive_with_data(50, hash, &dummy_filter_data(50)); - - assert_eq!(result, None); - assert_eq!(pipeline.filters_received, 0); - } - - #[test] - fn test_receive_batch_completion() { - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 2; - - // Set up a small batch (3 filters: 0, 1, 2) - pipeline.batch_trackers.insert(0, BatchTracker::new(2)); - pipeline.coordinator.mark_sent(&[0]); - - // Receive all filters - for h in 0..=2 { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - // Batch should be complete and moved to completed_batches - assert!(pipeline.batch_trackers.is_empty()); - assert_eq!(pipeline.completed_batches.len(), 1); - - let completed = pipeline.take_completed_batches(); - assert_eq!(completed.len(), 1); - let batch = completed.into_iter().next().unwrap(); - assert_eq!(batch.start_height(), 0); - assert_eq!(batch.end_height(), 2); - assert_eq!(batch.filters().len(), 3); - } - - #[test] - fn test_receive_out_of_order() { - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 4; - - pipeline.batch_trackers.insert(0, BatchTracker::new(4)); - pipeline.coordinator.mark_sent(&[0]); - - // Receive out of order - for h in [3, 1, 4, 0, 2] { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - // Should complete successfully - assert!(pipeline.batch_trackers.is_empty()); - assert_eq!(pipeline.completed_batches.len(), 1); - } - - #[test] - fn test_receive_updates_counters() { - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 99; - - pipeline.batch_trackers.insert(0, BatchTracker::new(99)); - pipeline.coordinator.mark_sent(&[0]); - - // Receive some filters - for h in [10, 5, 20, 15] { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - assert_eq!(pipeline.filters_received, 4); - assert_eq!(pipeline.highest_received, 20); - } - - #[test] - fn test_receive_small_batch_at_target() { - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 1005; - - // Small batch of 6 filters (1000-1005) - pipeline.batch_trackers.insert(1000, BatchTracker::new(1005)); - pipeline.coordinator.mark_sent(&[1000]); - - // Receive all 6 filters - for h in 1000..=1005 { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - assert_eq!(pipeline.completed_batches.len(), 1); - let batch = pipeline.completed_batches.iter().next().unwrap(); - assert_eq!(batch.filters().len(), 6); - } - - #[test] - fn test_receive_multiple_batches() { - let mut pipeline = FiltersPipeline::new(); - pipeline.target_height = 9; - - // Set up two batches manually - pipeline.batch_trackers.insert(0, BatchTracker::new(4)); - pipeline.batch_trackers.insert(5, BatchTracker::new(9)); - pipeline.coordinator.mark_sent(&[0, 5]); - - // Receive first batch - for h in 0..=4 { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - assert_eq!(pipeline.completed_batches.len(), 1); - assert_eq!(pipeline.batch_trackers.len(), 1); - - // Receive second batch - for h in 5..=9 { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - assert_eq!(pipeline.completed_batches.len(), 2); - assert!(pipeline.batch_trackers.is_empty()); - } - - // ========================================================================= - // find_batch_for_height Tests - // ========================================================================= - - #[test] - fn test_find_batch_for_height_found() { - let mut pipeline = FiltersPipeline::new(); - pipeline.batch_trackers.insert(0, BatchTracker::new(999)); - pipeline.batch_trackers.insert(1000, BatchTracker::new(1999)); - - assert_eq!(pipeline.find_batch_for_height(500), Some(0)); - assert_eq!(pipeline.find_batch_for_height(1500), Some(1000)); - } - - #[test] - fn test_find_batch_for_height_none() { - let mut pipeline = FiltersPipeline::new(); - pipeline.batch_trackers.insert(100, BatchTracker::new(199)); - - // Below range - assert_eq!(pipeline.find_batch_for_height(50), None); - // Above range - assert_eq!(pipeline.find_batch_for_height(250), None); - } - - #[test] - fn test_find_batch_for_height_boundary() { - let mut pipeline = FiltersPipeline::new(); - pipeline.batch_trackers.insert(100, BatchTracker::new(199)); - - // First height in batch - assert_eq!(pipeline.find_batch_for_height(100), Some(100)); - // Last height in batch - assert_eq!(pipeline.find_batch_for_height(199), Some(100)); - } - - // ========================================================================= - // Timeout Tests - // ========================================================================= - - #[test] - fn test_handle_timeouts_no_batches() { - let mut pipeline = FiltersPipeline::new(); - pipeline.handle_timeouts(); - } - - #[test] - fn test_handle_timeouts_requeue() { - let mut pipeline = create_pipeline_with_short_timeout(); - pipeline.target_height = 999; - - // Set up batch and mark as in-flight (simulating a sent request) - pipeline.batch_trackers.insert(0, BatchTracker::new(999)); - pipeline.coordinator.mark_sent(&[0]); - - // Wait for timeout - std::thread::sleep(Duration::from_millis(5)); - - pipeline.handle_timeouts(); - - // Batch should be re-queued in coordinator's pending queue - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert_eq!(pipeline.coordinator.active_count(), 0); - } - - #[test] - fn test_handle_timeouts_keeps_tracker() { - let mut pipeline = create_pipeline_with_short_timeout(); - pipeline.target_height = 99; - - pipeline.batch_trackers.insert(0, BatchTracker::new(99)); - pipeline.coordinator.mark_sent(&[0]); - - // Receive some filters before timeout - for h in 0..10 { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - std::thread::sleep(Duration::from_millis(5)); - - pipeline.handle_timeouts(); - - // Should timeout but tracker is preserved for late arrivals - assert!(pipeline.batch_trackers.contains_key(&0)); - assert_eq!(pipeline.batch_trackers.get(&0).unwrap().received(), 10); - } - - #[test] - fn test_timeout_does_not_duplicate_inflight_batches() { - // This test verifies the bug fix: when an early batch times out, - // only that batch is re-queued, not later in-flight batches. - let mut pipeline = FiltersPipeline { - coordinator: DownloadCoordinator::new( - DownloadConfig::default() - .with_timeout(Duration::from_millis(1)) - .with_max_concurrent(10), - ), - batch_trackers: HashMap::new(), - completed_batches: BTreeSet::new(), - target_height: 2999, - filters_received: 0, - highest_received: 0, - }; - - // Simulate 3 in-flight batches: 0-999, 1000-1999, 2000-2999 - pipeline.batch_trackers.insert(0, BatchTracker::new(999)); - pipeline.batch_trackers.insert(1000, BatchTracker::new(1999)); - pipeline.batch_trackers.insert(2000, BatchTracker::new(2999)); - pipeline.coordinator.mark_sent(&[0, 1000, 2000]); - - assert_eq!(pipeline.coordinator.active_count(), 3); - assert_eq!(pipeline.coordinator.pending_count(), 0); - - // Wait for timeout - std::thread::sleep(Duration::from_millis(5)); - - // Handle timeouts - all 3 should timeout and be re-queued - pipeline.handle_timeouts(); - - // All 3 batches should be in the pending queue, not duplicated - assert_eq!(pipeline.coordinator.pending_count(), 3); - assert_eq!(pipeline.coordinator.active_count(), 0); - - // Take pending items - should get exactly 3, not more - let pending = pipeline.coordinator.take_pending(10); - assert_eq!(pending.len(), 3); - assert!(pending.contains(&0)); - assert!(pending.contains(&1000)); - assert!(pending.contains(&2000)); - } - - // ========================================================================= - // send_pending Tests - // ========================================================================= - - #[tokio::test] - async fn test_send_pending_single_batch() { - let headers = Header::dummy_batch(0..1000); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 999); - - let (sender, mut rx) = create_test_request_sender(); - - let count = pipeline.send_pending(&sender, &storage).await.unwrap(); - - assert_eq!(count, 1); - assert_eq!(pipeline.coordinator.active_count(), 1); - assert!(pipeline.batch_trackers.contains_key(&0)); - // No more pending since the single batch was sent - assert_eq!(pipeline.coordinator.pending_count(), 0); - - // Verify message was sent - let request = rx.try_recv().unwrap(); - let NetworkRequest::SendMessage(msg) = request else { - panic!("Expected SendMessage variant"); - }; - if let NetworkMessage::GetCFilters(gcf) = msg { - assert_eq!(gcf.start_height, 0); - assert_eq!(gcf.filter_type, 0); - } else { - panic!("Expected GetCFilters message"); - } - } - - #[tokio::test] - async fn test_send_pending_respects_limit() { - // Create enough headers for many batches - let headers = Header::dummy_batch(0..25000); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 24999); - - let (sender, _rx) = create_test_request_sender(); - - let count = pipeline.send_pending(&sender, &storage).await.unwrap(); - - // 25 batches needed, but only 20 can be in-flight at once - assert_eq!(count, MAX_CONCURRENT_FILTER_BATCHES); - assert_eq!(pipeline.coordinator.active_count(), MAX_CONCURRENT_FILTER_BATCHES); - assert_eq!(pipeline.batch_trackers.len(), 25); - assert_eq!(pipeline.coordinator.pending_count(), 5); - } - - #[tokio::test] - async fn test_send_pending_calculates_end() { - let headers = Header::dummy_batch(0..1500); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = FiltersPipeline::new(); - // Target is 1200, so second batch ends at 1200 not 1999 - pipeline.init(0, 1200); - - let (sender, _rx) = create_test_request_sender(); - - let count = pipeline.send_pending(&sender, &storage).await.unwrap(); - - assert_eq!(count, 2); - - // First batch: 0-999 - assert!(pipeline.batch_trackers.contains_key(&0)); - assert_eq!(pipeline.batch_trackers.get(&0).unwrap().end_height(), 999); - - // Second batch: 1000-1200 (capped by target) - assert!(pipeline.batch_trackers.contains_key(&1000)); - assert_eq!(pipeline.batch_trackers.get(&1000).unwrap().end_height(), 1200); - } - - #[tokio::test] - async fn test_send_pending_sends_all_queued() { - let headers = Header::dummy_batch(0..3000); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 2500); - - let (sender, _rx) = create_test_request_sender(); - - let count = pipeline.send_pending(&sender, &storage).await.unwrap(); - - // Should send all 3 batches: 0-999, 1000-1999, 2000-2500 - assert_eq!(count, 3); - assert_eq!(pipeline.coordinator.active_count(), 3); - assert_eq!(pipeline.coordinator.pending_count(), 0); - } - - #[tokio::test] - async fn test_send_pending_no_work_when_queue_empty() { - let headers = Header::dummy_batch(0..100); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 50); - - let (sender, _rx) = create_test_request_sender(); - - // First send exhausts the queue - let count = pipeline.send_pending(&sender, &storage).await.unwrap(); - assert_eq!(count, 1); - - // Second send has nothing to do - let count = pipeline.send_pending(&sender, &storage).await.unwrap(); - assert_eq!(count, 0); - } - - // ========================================================================= - // Integration Tests - // ========================================================================= - - #[tokio::test] - async fn test_full_batch_lifecycle() { - let headers = Header::dummy_batch(0..100); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 99); - - let (sender, _rx) = create_test_request_sender(); - - // Send request - let sent = pipeline.send_pending(&sender, &storage).await.unwrap(); - assert_eq!(sent, 1); - assert_eq!(pipeline.coordinator.active_count(), 1); - - // Receive all filters - for h in 0..=99 { - let hash = Header::dummy(h).block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - - // Batch should be complete - assert_eq!(pipeline.coordinator.active_count(), 0); - assert_eq!(pipeline.completed_batches.len(), 1); - assert_eq!(pipeline.filters_received, 100); - assert_eq!(pipeline.highest_received, 99); - - // Take completed - let completed = pipeline.take_completed_batches(); - assert_eq!(completed.len(), 1); - assert!(pipeline.completed_batches.is_empty()); - } - - #[tokio::test] - async fn test_timeout_and_retry_flow() { - let headers = Header::dummy_batch(0..1000); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = create_pipeline_with_short_timeout(); - pipeline.init(0, 999); - - let (sender, _rx) = create_test_request_sender(); - - // Send initial request - pipeline.send_pending(&sender, &storage).await.unwrap(); - assert_eq!(pipeline.coordinator.active_count(), 1); - assert_eq!(pipeline.coordinator.pending_count(), 0); - - // Wait for timeout - std::thread::sleep(Duration::from_millis(5)); - - // Handle timeout - should re-queue the batch via coordinator - pipeline.handle_timeouts(); - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert_eq!(pipeline.coordinator.active_count(), 0); - - // Tracker should still exist for late arrivals - assert!(pipeline.batch_trackers.contains_key(&0)); - - // Can retry by sending again - pipeline.send_pending(&sender, &storage).await.unwrap(); - assert_eq!(pipeline.coordinator.active_count(), 1); - - // Existing tracker is reused (not replaced) - assert!(pipeline.batch_trackers.contains_key(&0)); - } - - #[test] - fn test_take_completed_batches_clears() { - let mut pipeline = FiltersPipeline::new(); - - // Add some completed batches - pipeline.completed_batches.insert(FiltersBatch::new(0, 99, HashMap::new())); - pipeline.completed_batches.insert(FiltersBatch::new(100, 199, HashMap::new())); - - let taken = pipeline.take_completed_batches(); - assert_eq!(taken.len(), 2); - assert!(pipeline.completed_batches.is_empty()); - } - - #[test] - fn test_filters_batch_filters_mut() { - let mut batch = FiltersBatch::new(0, 0, HashMap::new()); - - batch - .filters_mut() - .insert(FilterMatchKey::new(0, BlockHash::all_zeros()), BlockFilter::new(&[0x01])); - - assert_eq!(batch.filters().len(), 1); - } - - #[tokio::test] - async fn test_deferred_batch_keeps_end_height_after_extend() { - // init(0, 2500) creates 3 batches but only 2 can be sent (max concurrent=2). - // The boundary batch (2000-2500) stays queued. After extend_target changes - // target_height to 4000, the deferred batch must still use end_height=2500. - let headers = Header::dummy_batch(0..5000); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = create_pipeline_with_low_concurrency(); - pipeline.init(0, 2500); - assert_eq!(pipeline.coordinator.pending_count(), 3); // 0, 1000, 2000 - - let (sender, _rx) = create_test_request_sender(); - - // Only 2 batches sent, batch 2000 stays queued - pipeline.send_pending(&sender, &storage).await.unwrap(); - assert_eq!(pipeline.coordinator.active_count(), 2); - assert_eq!(pipeline.coordinator.pending_count(), 1); - - // Extend target — batch 2000's tracker must keep end_height=2500 - pipeline.extend_target(4000); - - // Complete batch 0 to free a slot, then send deferred batch - for h in 0..1000 { - let hash = headers[h as usize].block_hash(); - pipeline.receive_with_data(h, hash, &dummy_filter_data(h)); - } - pipeline.send_pending(&sender, &storage).await.unwrap(); - - assert_eq!( - pipeline.batch_trackers.get(&2000).unwrap().end_height(), - 2500, - "deferred batch should use its original end height" - ); - } - - #[tokio::test] - async fn test_send_pending_requeues_on_missing_header() { - // Headers 0..999 exist, but NOT 1999 (stop hash for batch 1000-1999). - let headers = Header::dummy_batch(0..1000); - let tmp_dir = TempDir::new().unwrap(); - let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - storage - .store_headers( - &headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let mut pipeline = FiltersPipeline::new(); - pipeline.init(0, 1999); - assert_eq!(pipeline.coordinator.pending_count(), 2); - - let (sender, mut rx) = create_test_request_sender(); - - // Batch 0 succeeds (header 999 exists), batch 1000 re-queued (header 1999 missing) - let sent = pipeline.send_pending(&sender, &storage).await.unwrap(); - assert_eq!(sent, 1); - assert_eq!(pipeline.coordinator.active_count(), 1); - assert_eq!(pipeline.coordinator.pending_count(), 1); - - let request = rx.try_recv().unwrap(); - match request { - NetworkRequest::SendMessage(NetworkMessage::GetCFilters(gcf)) => { - assert_eq!(gcf.start_height, 0); - } - other => panic!("Expected GetCFilters, got {:?}", other), - } - assert!(rx.try_recv().is_err(), "should not have sent second request"); - - // Store the missing headers and retry - let more_headers = Header::dummy_batch(1000..2000); - storage - .store_headers( - &more_headers.iter().map(crate::types::HashedBlockHeader::from).collect::>(), - ) - .await - .unwrap(); - - let sent = pipeline.send_pending(&sender, &storage).await.unwrap(); - assert_eq!(sent, 1); - assert_eq!(pipeline.coordinator.active_count(), 2); - assert_eq!(pipeline.coordinator.pending_count(), 0); - } } diff --git a/dash-spv/src/sync/filters/progress.rs b/dash-spv/src/sync/filters/progress.rs index e38cba4ea..10472e7f0 100644 --- a/dash-spv/src/sync/filters/progress.rs +++ b/dash-spv/src/sync/filters/progress.rs @@ -146,15 +146,14 @@ impl FiltersProgress { impl fmt::Display for FiltersProgress { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let pct = self.percentage() * 100.0; write!( f, - "{:?} {}/{} ({:.1}%) stored:{}, downloaded: {}, processed: {}, matched: {}, last_activity: {}s", + "{:?} {}/{} ({:.1}%) committed: {}, downloaded: {}, processed: {}, matched: {}, last_activity: {}s", self.state, + self.current_height(), + self.target_height(), + self.percentage() * 100.0, self.committed_height, - self.target_height, - pct, - self.stored_height, self.downloaded, self.processed, self.matched, @@ -167,7 +166,15 @@ impl ProgressPercentage for FiltersProgress { fn target_height(&self) -> u32 { self.target_height } + /// How far the FILTERS themselves are down: the height up to which they have + /// been downloaded, verified and stored. + /// + /// Not `committed_height`, which is the scan/commit frontier — that trails the + /// download by a long way (it waits on the matched blocks and the gap-limit + /// rescan), so reporting it made the filters look stalled while they were in + /// fact streaming in, and hid the fact that they download in parallel with the + /// filter headers. `committed_height` is still reported in the detail fields. fn current_height(&self) -> u32 { - self.committed_height + self.stored_height } } diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index ecce171fa..1d651f3ac 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::{SyncError, SyncResult}; -use crate::network::{Message, MessageType, RequestSender}; +use crate::network2::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage, FilterStorage}; use crate::sync::sync_manager::ensure_not_started; use crate::sync::{ @@ -8,6 +8,8 @@ use crate::sync::{ use async_trait::async_trait; use dashcore::network::message::NetworkMessage; use key_wallet_manager::WalletInterface; +use std::net::SocketAddr; +use std::sync::Arc; #[async_trait] impl< @@ -33,21 +35,27 @@ impl< self.progress.update_target_height(height); } - fn wanted_message_types(&self) -> &'static [MessageType] { - &[MessageType::CFilter] + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { + &[crate::network2::MessageType::CFilter] } - /// Keep `active_batches`, the block-match tracker, pending verified - /// batches, and the filter pipeline's per-batch trackers. Move in-flight - /// `getcfilters` slots back to pending so the next `send_pending` reissues - /// them to the new peer immediately. Without this preservation, a re-scan - /// after reconnect would re-track the same block hashes and leak - /// `pending_blocks` counters that never reach zero. - fn on_disconnect(&mut self) { - self.filter_pipeline.requeue_in_flight(); + fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { + if let crate::network2::RequestKey::CFilters(start_height) = key { + self.filter_pipeline.mark_on_flight(*start_height); + } } - async fn start_sync(&mut self, requests: &RequestSender) -> SyncResult> { + /// Keep `active_batches`, the block-match tracker, pending verified + /// batches, and the filter pipeline's per-batch trackers across the + /// disconnect. In-flight `getcfilters` slots are left untouched; reissuing + /// requests dropped by a disconnect is deferred to the future + /// network-manager retry layer. + fn on_disconnect(&mut self) {} + + async fn start_sync( + &mut self, + network: &Arc, + ) -> SyncResult> { ensure_not_started(self.state(), self.identifier())?; // Resume in-progress work preserved across a disconnect cycle. @@ -56,7 +64,7 @@ impl< // insert a fresh batch at `scan_start` and clobber the existing one, // leaking its `pending_blocks` counter forever. if !self.active_batches.is_empty() { - self.filter_pipeline.send_pending(requests, &*self.header_storage.read().await).await?; + self.filter_pipeline.send_pending(network, &*self.header_storage.read().await).await?; self.set_state(SyncState::Syncing); return Ok(vec![]); } @@ -76,7 +84,7 @@ impl< let mut events = vec![SyncEvent::SyncStart { identifier: self.identifier(), }]; - events.extend(self.start_download(requests).await?); + events.extend(self.start_download(network).await?); return Ok(events); } @@ -86,7 +94,7 @@ impl< // above this must not emit a SyncStart. if stored_filters_tip > 0 && stored_filters_tip == self.progress.committed_height() { self.progress.update_filter_header_tip_height(stored_filters_tip); - return self.start_download(requests).await; + return self.start_download(network).await; } // No stored filters to process - wait for FilterHeadersSyncComplete events @@ -96,21 +104,24 @@ impl< async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult> { - let NetworkMessage::CFilter(cfilter) = msg.inner() else { + let NetworkMessage::CFilter(cfilter) = &msg else { return Ok(vec![]); }; // Find height for this filter + let t0 = std::time::Instant::now(); let height = self.header_storage.read().await.get_header_height_by_hash(&cfilter.block_hash).await?; + crate::timer::P_HEIGHT_LOOKUP.add(t0.elapsed()); let Some(h) = height else { tracing::warn!( block_hash = %cfilter.block_hash, - peer = %msg.peer_address(), + peer = %peer, "Received CFilter for unknown block hash, rejecting as invalid" ); // TODO: should we penalize the peer a bit? @@ -121,33 +132,54 @@ impl< }; // Buffer filter in pipeline - self.filter_pipeline.receive_with_data(h, cfilter.block_hash, &cfilter.filter); + let t1 = std::time::Instant::now(); + let batch_completed = + self.filter_pipeline.receive_with_data(h, cfilter.block_hash, &cfilter.filter); + crate::timer::P_RECV_DATA.add(t1.elapsed()); + + // A completed batch == one `getcfilters` request fully answered: free that + // peer's in-flight unit (the reader skips per-`cfilter` decrements). + if batch_completed.is_some() { + network.request_completed(peer, 1).await; + } // Send more requests if there are free slots + let t2 = std::time::Instant::now(); let header_storage = self.header_storage.read().await; - self.filter_pipeline.send_pending(requests, &*header_storage).await?; + self.filter_pipeline.send_pending(network, &*header_storage).await?; drop(header_storage); + crate::timer::P_SEND_PENDING.add(t2.elapsed()); - Ok(self.store_and_match_batches().await?) + let t3 = std::time::Instant::now(); + let events = self.store_and_match_batches().await?; + crate::timer::P_STORE_MATCH.add(t3.elapsed()); + Ok(events) } async fn handle_sync_event( &mut self, event: &SyncEvent, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { match event { SyncEvent::FilterHeadersSyncComplete { tip_height, } => { - return self.handle_new_filter_headers(*tip_height, requests).await; + return self.handle_new_filter_headers(*tip_height, network).await; } SyncEvent::FilterHeadersStored { + start_height, tip_height, + headers, .. } => { - return self.handle_new_filter_headers(*tip_height, requests).await; + // The headers come VERIFIED (filter-headers validates the chain + // before notifying), and the filters they authenticate are already + // downloading — so cache them and verify those filters straight from + // memory, with no storage read. + self.cache_filter_headers(*start_height, headers); + return self.handle_new_filter_headers(*tip_height, network).await; } // React to BlockProcessed events from the BlocksManager @@ -196,7 +228,7 @@ impl< Ok(vec![]) } - async fn tick(&mut self, requests: &RequestSender) -> SyncResult> { + async fn tick(&mut self, network: &Arc) -> SyncResult> { // Detect a wallet that was added behind our scan progress and rescan // from its `synced_height`. Reset committed_height to the lowest // synced_height across the stale wallets only, so already-synced @@ -217,7 +249,7 @@ impl< ); self.reset_for_rescan(); self.progress.update_committed_height(stale_min_synced); - return self.start_download(requests).await; + return self.start_download(network).await; } } @@ -233,12 +265,12 @@ impl< return Ok(vec![]); } - // Handle timeouts - self.filter_pipeline.handle_timeouts(); - - // Send pending requests (decoupled from processing) + // Re-issue any timed-out GetCFilters, then send pending (decoupled from + // processing). Filters yield ~1000 messages per batch, so use a longer + // timeout than the 1:1 request phases. + self.filter_pipeline.handle_timeouts(std::time::Duration::from_secs(30)); let header_storage = self.header_storage.read().await; - self.filter_pipeline.send_pending(requests, &*header_storage).await?; + self.filter_pipeline.send_pending(network, &*header_storage).await?; drop(header_storage); // Store completed batches and do speculative matching diff --git a/dash-spv/src/sync/instantsend/manager.rs b/dash-spv/src/sync/instantsend/manager.rs index eb0ed5509..838646a19 100644 --- a/dash-spv/src/sync/instantsend/manager.rs +++ b/dash-spv/src/sync/instantsend/manager.rs @@ -281,179 +281,3 @@ impl std::fmt::Debug for InstantSendManager { .finish() } } -#[cfg(test)] -mod tests { - use super::*; - use crate::network::MessageType; - use crate::sync::{ManagerIdentifier, SyncManager, SyncManagerProgress, SyncState}; - use dashcore::bls_sig_utils::BLSSignature; - use dashcore::hash_types::CycleHash; - use dashcore::hashes::Hash; - use dashcore::OutPoint; - - fn create_test_instantlock(txid: Txid) -> InstantLock { - InstantLock { - version: 1, - inputs: vec![OutPoint::default()], - txid, - cyclehash: CycleHash::all_zeros(), - signature: BLSSignature::from([1u8; 96]), // Non-zero signature - } - } - - fn create_test_manager() -> InstantSendManager { - let engine = Arc::new(RwLock::new(MasternodeListEngine::default_for_network( - dashcore::Network::Testnet, - ))); - InstantSendManager::new(engine) - } - - #[tokio::test] - async fn test_instantsend_manager_new() { - let manager = create_test_manager(); - assert_eq!(manager.identifier(), ManagerIdentifier::InstantSend); - assert_eq!(manager.state(), SyncState::WaitForEvents); - assert_eq!(manager.wanted_message_types(), vec![MessageType::ISLock, MessageType::Inv]); - } - - /// Buffered `MasternodeStateUpdated` events delivered during - /// `WaitingForConnections` must not run validation or transition state. - /// `pending_instantlocks` is cleared on disconnect and - /// `MasternodesManager` re-emits the event after reconnect. - #[tokio::test] - async fn test_handle_sync_event_drops_masternode_state_updated_in_waiting_for_connections() { - use crate::network::RequestSender; - use crate::sync::SyncEvent; - use tokio::sync::mpsc::unbounded_channel; - - let mut manager = create_test_manager(); - manager.set_state(SyncState::WaitingForConnections); - - let event = SyncEvent::MasternodeStateUpdated { - height: 100, - qr_info_result: None, - }; - let (tx, _rx) = unbounded_channel(); - let events = manager.handle_sync_event(&event, &RequestSender::new(tx)).await.unwrap(); - - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - } - - #[tokio::test] - async fn test_instantsend_duplicate_handling() { - let mut manager = create_test_manager(); - - let txid = Txid::from_byte_array([1u8; 32]); - let islock1 = create_test_instantlock(txid); - let islock2 = create_test_instantlock(txid); - - // First should process - let events1 = manager.process_instantlock(&islock1).await.unwrap(); - assert_eq!(events1.len(), 1); - - // Second should be ignored as duplicate - let events2 = manager.process_instantlock(&islock2).await.unwrap(); - assert_eq!(events2.len(), 0); - } - - #[tokio::test] - async fn test_instantsend_pending_queue() { - let mut manager = create_test_manager(); - - // Without masternode engine, InstantLocks should be queued - let txid = Txid::from_byte_array([1u8; 32]); - let islock = create_test_instantlock(txid); - let _ = manager.process_instantlock(&islock).await.unwrap(); - - assert_eq!(manager.pending_count(), 1); - } - - #[tokio::test] - async fn test_instantsend_structural_validation() { - let manager = create_test_manager(); - - // Valid structure - let txid = Txid::from_byte_array([1u8; 32]); - let valid = create_test_instantlock(txid); - assert!(manager.validate_structure(&valid)); - - // Empty inputs - let mut invalid = create_test_instantlock(txid); - invalid.inputs = vec![]; - assert!(!manager.validate_structure(&invalid)); - - // Null txid - let invalid_txid = InstantLock { - version: 1, - inputs: vec![OutPoint::default()], - txid: Txid::all_zeros(), - cyclehash: CycleHash::all_zeros(), - signature: BLSSignature::from([1u8; 96]), - }; - assert!(!manager.validate_structure(&invalid_txid)); - - // Zeroed signature - let invalid_sig = InstantLock { - version: 1, - inputs: vec![OutPoint::default()], - txid: Txid::from_byte_array([1u8; 32]), - cyclehash: CycleHash::all_zeros(), - signature: BLSSignature::from([0u8; 96]), - }; - assert!(!manager.validate_structure(&invalid_sig)); - } - - #[tokio::test] - async fn test_instantsend_progress() { - let mut manager = create_test_manager(); - manager.set_state(SyncState::Syncing); - manager.progress.update_pending(2); - manager.progress.add_valid(8); - manager.progress.add_invalid(2); - - let progress = manager.progress(); - if let SyncManagerProgress::InstantSend(progress) = progress { - assert_eq!(progress.state(), SyncState::Syncing); - assert_eq!(progress.valid(), 8); - assert_eq!(progress.invalid(), 2); - assert_eq!(progress.pending(), 2); - assert!(progress.last_activity().elapsed().as_secs() < 1); - } else { - panic!("Expected SyncManagerProgress::InstantSend"); - } - } - - #[tokio::test] - async fn test_instantsend_accessors() { - let mut manager = create_test_manager(); - - let txid = Txid::from_byte_array([1u8; 32]); - let islock = create_test_instantlock(txid); - let _ = manager.process_instantlock(&islock).await.unwrap(); - - // Should be retrievable by txid - assert!(manager.get_instantlock(&txid).is_some()); - - // Unknown txid - let unknown = Txid::from_byte_array([2u8; 32]); - assert!(manager.get_instantlock(&unknown).is_none()); - } - - #[tokio::test] - async fn test_instantsend_cache_limit() { - let mut manager = create_test_manager(); - - // Add more than MAX_CACHE_SIZE instantlocks - for i in 0..MAX_CACHE_SIZE + 10 { - let mut bytes = [0u8; 32]; - bytes[0..4].copy_from_slice(&(i as u32).to_le_bytes()); - let txid = Txid::from_byte_array(bytes); - let islock = create_test_instantlock(txid); - let _ = manager.process_instantlock(&islock).await.unwrap(); - } - - // Should be capped at MAX_CACHE_SIZE - assert!(manager.cached_count() <= MAX_CACHE_SIZE); - } -} diff --git a/dash-spv/src/sync/instantsend/sync_manager.rs b/dash-spv/src/sync/instantsend/sync_manager.rs index e9ee4e96e..6abc5e1b5 100644 --- a/dash-spv/src/sync/instantsend/sync_manager.rs +++ b/dash-spv/src/sync/instantsend/sync_manager.rs @@ -1,11 +1,13 @@ use crate::error::SyncResult; -use crate::network::{Message, MessageType, RequestSender}; +use crate::network2::PeerNetworkManager; use crate::sync::{ InstantSendManager, ManagerIdentifier, SyncEvent, SyncManager, SyncManagerProgress, SyncState, }; use async_trait::async_trait; use dashcore::network::message::NetworkMessage; use dashcore::network::message_blockdata::Inventory; +use std::net::SocketAddr; +use std::sync::Arc; #[async_trait] impl SyncManager for InstantSendManager { @@ -21,8 +23,9 @@ impl SyncManager for InstantSendManager { self.progress.set_state(state); } - fn wanted_message_types(&self) -> &'static [MessageType] { - &[MessageType::ISLock, MessageType::Inv] + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { + use crate::network2::MessageType; + &[MessageType::IsDLock, MessageType::Inv] } fn on_disconnect(&mut self) { @@ -31,10 +34,11 @@ impl SyncManager for InstantSendManager { async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult> { - match msg.inner() { + match &msg { NetworkMessage::ISLock(instantlock) => self.process_instantlock(instantlock).await, NetworkMessage::Inv(inv) => { // Check for InstantSendLock inventory items @@ -49,7 +53,7 @@ impl SyncManager for InstantSendManager { "Received {} InstantSendLock announcements, requesting via getdata", islocks_to_request.len() ); - requests.request_inventory(islocks_to_request, msg.peer_address())?; + network.send(NetworkMessage::GetData(islocks_to_request)).await; } Ok(vec![]) } @@ -60,7 +64,7 @@ impl SyncManager for InstantSendManager { async fn handle_sync_event( &mut self, event: &SyncEvent, - _requests: &RequestSender, + _network: &Arc, ) -> SyncResult> { // Drop buffered events that arrive between `stop_sync` and the next // `start_sync`. `pending_instantlocks` is cleared on disconnect, and @@ -100,7 +104,7 @@ impl SyncManager for InstantSendManager { Ok(vec![]) } - async fn tick(&mut self, _requests: &RequestSender) -> SyncResult> { + async fn tick(&mut self, _network: &Arc) -> SyncResult> { // Prune old entries periodically self.prune_old_entries(); Ok(vec![]) diff --git a/dash-spv/src/sync/masternodes/manager.rs b/dash-spv/src/sync/masternodes/manager.rs index b31101586..c8003977d 100644 --- a/dash-spv/src/sync/masternodes/manager.rs +++ b/dash-spv/src/sync/masternodes/manager.rs @@ -13,10 +13,11 @@ use tokio::sync::RwLock; use super::pipeline::MnListDiffPipeline; use crate::error::{SyncError, SyncResult}; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::storage::BlockHeaderStorage; use crate::sync::{MasternodesProgress, SyncEvent, SyncManager, SyncState}; -use dashcore::network::message_qrinfo::QRInfo; +use dashcore::network::message::NetworkMessage; +use dashcore::network::message_qrinfo::{GetQRInfo, QRInfo}; use dashcore::BlockHash; use std::collections::BTreeSet; @@ -392,7 +393,7 @@ impl MasternodesManager { /// lightweight completion path when the response drains the pipeline. pub(super) async fn send_tip_mnlistdiff_update( &mut self, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { let new_tip_hash = { let storage = self.header_storage.read().await; @@ -414,7 +415,7 @@ impl MasternodesManager { self.sync_state.pipeline_mode = PipelineMode::Incremental; self.sync_state.mnlistdiff_pipeline.queue_requests(vec![(base_hash, new_tip_hash)]); - self.sync_state.mnlistdiff_pipeline.send_pending(requests)?; + self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; Ok(vec![]) } @@ -439,7 +440,7 @@ impl MasternodesManager { /// would have done had the intermediate events not been dropped. pub(super) async fn complete_pipeline( &mut self, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { match std::mem::take(&mut self.sync_state.pipeline_mode) { PipelineMode::QuorumValidation { @@ -457,7 +458,7 @@ impl MasternodesManager { ); self.sync_state.qrinfo_retry_count = 0; self.sync_state.clear_pending(); - match self.send_qrinfo_for_tip(requests).await { + match self.send_qrinfo_for_tip(network).await { Ok(extra) => events.extend(extra), Err(e) => tracing::warn!( error = %e, @@ -509,7 +510,7 @@ impl MasternodesManager { /// Called when BlockHeaderSyncComplete is received, ensuring we have all headers. pub(super) async fn send_qrinfo_for_tip( &mut self, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { // Get info from storage let (tip_height, tip_block_hash) = { @@ -545,7 +546,13 @@ impl MasternodesManager { // connected during a reconnect race), the `?` propagates and we leave // `WaitingForConnections` intact instead of stranding the manager in // `Syncing` with `qrinfo_in_flight = None`, which `tick` cannot recover. - requests.request_qr_info(base_hashes, tip_block_hash, true)?; + network + .send(NetworkMessage::GetQRInfo(GetQRInfo { + base_block_hashes: base_hashes, + block_request_hash: tip_block_hash, + extra_share: true, + })) + .await; self.progress.add_qr_infos_requested(1); self.sync_state.record_qrinfo_attempt(tip_height); self.sync_state.start_waiting_for_qrinfo(tip_block_hash); @@ -614,404 +621,3 @@ impl std::fmt::Debug for MasternodesManager { f.debug_struct("MasternodesManager").field("progress", &self.progress).finish() } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::network::{MessageType, NetworkRequest}; - use crate::storage::{DiskStorageManager, PersistentBlockHeaderStorage, StorageManager}; - use crate::sync::sync_manager::SyncManager; - use crate::sync::{ManagerIdentifier, SyncManagerProgress}; - use dashcore::block::Header; - use dashcore::hashes::Hash; - use dashcore::network::message::NetworkMessage; - use dashcore::sml::masternode_list::MasternodeList; - use tokio::sync::mpsc; - - type TestMasternodesManager = MasternodesManager; - - async fn create_test_manager_for(network: dashcore::Network) -> TestMasternodesManager { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let engine = Arc::new(RwLock::new(MasternodeListEngine::default_for_network(network))); - MasternodesManager::new(storage.block_headers(), engine, network).await - } - - async fn create_test_manager() -> TestMasternodesManager { - create_test_manager_for(dashcore::Network::Testnet).await - } - - /// Build a regtest manager whose engine has a single list at `tip` and - /// whose block header storage is populated with dummy headers up to - /// `tip`, in `Synced` state with `pipeline_mode = Incremental` and - /// `block_header_tip_height = tip`. Storage must be populated so that - /// `send_qrinfo_for_tip` finds a tip and reaches the network dispatch; - /// otherwise it short-circuits at `storage.get_tip()` and the catch-up - /// path can't be observed at the network layer. Returns the manager, a - /// `RequestSender`, and the matching receiver so the caller binds it - /// (the channel closes when the receiver drops). - async fn make_synced_incremental_manager( - tip: u32, - ) -> (TestMasternodesManager, RequestSender, mpsc::UnboundedReceiver) { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let block_headers = storage.block_headers(); - block_headers - .write() - .await - .store_headers( - &Header::dummy_batch(0..tip + 1) - .iter() - .map(crate::types::HashedBlockHeader::from) - .collect::>(), - ) - .await - .unwrap(); - let engine = engine_with_lists(&[(tip, 1)]); - let mut manager = MasternodesManager::new( - block_headers, - Arc::new(RwLock::new(engine)), - dashcore::Network::Regtest, - ) - .await; - manager.set_state(SyncState::Synced); - manager.sync_state.pipeline_mode = PipelineMode::Incremental; - manager.progress.update_block_header_tip_height(tip); - let (tx, rx) = mpsc::unbounded_channel(); - (manager, RequestSender::new(tx), rx) - } - - #[tokio::test] - async fn test_masternode_manager_new() { - let manager = create_test_manager().await; - assert_eq!(manager.identifier(), ManagerIdentifier::Masternode); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - assert_eq!( - manager.wanted_message_types(), - vec![MessageType::MnListDiff, MessageType::QRInfo] - ); - } - - #[tokio::test] - async fn test_masternode_manager_progress() { - let mut manager = create_test_manager().await; - manager.progress.update_current_height(500); - manager.progress.update_target_height(1000); - manager.progress.add_diffs_processed(10); - manager.progress.add_qr_infos_requested(3); - manager.progress.add_validated_cycles(2); - manager.progress.add_rotation_cycles(4); - - let progress = manager.progress(); - if let SyncManagerProgress::Masternodes(progress) = progress { - assert_eq!(progress.current_height(), 500); - assert_eq!(progress.target_height(), 1000); - assert_eq!(progress.diffs_processed(), 10); - assert_eq!(progress.qr_infos_requested(), 3); - assert_eq!(progress.validated_cycles(), 2); - assert_eq!(progress.rotation_cycles(), 4); - assert!(progress.last_activity().elapsed().as_secs() < 1); - } else { - panic!("Expected SyncManagerProgress::Masternodes"); - } - } - - fn anchor_hash(n: u8) -> BlockHash { - BlockHash::from_byte_array([n; 32]) - } - - fn engine_with_lists(lists: &[(u32, u8)]) -> MasternodeListEngine { - let mut engine = MasternodeListEngine::default_for_network(dashcore::Network::Regtest); - for (height, tag) in lists { - engine - .masternode_lists - .insert(*height, MasternodeList::empty(anchor_hash(*tag), *height)); - } - engine - } - - // Regtest `isd_llmq_type` is `LlmqtypeTestDIP0024` which uses `DKG_TEST` with - // `interval=24`, so `max_anchor_height = tip_cycle_start - 4 * 24`. - #[test] - fn test_compute_qrinfo_anchor_hash() { - struct Case { - name: &'static str, - lists: &'static [(u32, u8)], - tip: u32, - expect: Option, - } - let cases = [ - Case { - name: "empty engine", - lists: &[], - tip: 200, - expect: None, - }, - Case { - name: "tip too low, anchor underflows", - lists: &[(0, 1)], - tip: 50, - expect: None, - }, - Case { - name: "no stored list old enough", - lists: &[(100, 1), (150, 2)], - tip: 200, - expect: None, - }, - Case { - name: "single list exactly at max_anchor_height", - lists: &[(96, 1)], - tip: 200, - expect: Some(1), - }, - Case { - name: "picks highest list at or below max_anchor_height", - lists: &[(50, 1), (80, 2), (100, 3)], - tip: 200, - expect: Some(2), - }, - Case { - name: "mid-cycle tip rounds down to cycle start", - lists: &[(96, 1)], - tip: 215, - expect: Some(1), - }, - ]; - for case in &cases { - let engine = engine_with_lists(case.lists); - let got = compute_qrinfo_anchor_hash(&engine, dashcore::Network::Regtest, case.tip); - assert_eq!(got, case.expect.map(anchor_hash), "case: {}", case.name); - } - } - - // Regtest `isd_llmq_type` uses `DKG_TEST_DIP0024` (`interval=24`, - // `mining_window=[12, 20]`). Cycle 48 → window `[60, 68]`; cycle 72 → - // window `[84, 92]`. - #[tokio::test] - async fn test_next_pipeline_mode_fires_qrinfo_once_per_tip() { - let mut manager = create_test_manager_for(dashcore::Network::Regtest).await; - - // First call inside cycle 48's window picks QuorumValidation. The - // cycle rollover from None into cycle 48 also bumps `rotation_cycles` - // to 1. Per-tip state is not yet bumped because `next_pipeline_mode` - // is purely a decider. The caller bumps state via `record_qrinfo_attempt` - // when it actually fires. - assert!(matches!(manager.next_pipeline_mode(60), PipelineMode::QuorumValidation { .. })); - assert_eq!(manager.sync_state.current_cycle_attempts, 0); - assert_eq!(manager.sync_state.last_window_qrinfo_tip, None); - assert_eq!(manager.progress.rotation_cycles(), 1); - - // Simulate the caller firing the QRInfo. - manager.sync_state.record_qrinfo_attempt(60); - assert_eq!(manager.sync_state.current_cycle_attempts, 1); - assert_eq!(manager.sync_state.last_window_qrinfo_tip, Some(60)); - - // Re-entering with the same tip after a fire falls through to - // Incremental. The per-tip gate prevents refiring and the cycle is - // unchanged. - assert!(matches!(manager.next_pipeline_mode(60), PipelineMode::Incremental)); - assert_eq!(manager.sync_state.current_cycle_attempts, 1); - assert_eq!(manager.progress.rotation_cycles(), 1); - - // A new tip inside the same window picks QuorumValidation again. - assert!(matches!(manager.next_pipeline_mode(61), PipelineMode::QuorumValidation { .. })); - manager.sync_state.record_qrinfo_attempt(61); - assert_eq!(manager.sync_state.current_cycle_attempts, 2); - assert_eq!(manager.sync_state.last_window_qrinfo_tip, Some(61)); - - // Same tip again: still Incremental. - assert!(matches!(manager.next_pipeline_mode(61), PipelineMode::Incremental)); - assert_eq!(manager.sync_state.current_cycle_attempts, 2); - - // Cycle rollover to cycle 72 resets the per-tip gate, so the first tip - // inside the new window picks QuorumValidation and `rotation_cycles` - // bumps to 2. - assert!(matches!(manager.next_pipeline_mode(84), PipelineMode::QuorumValidation { .. })); - assert_eq!(manager.sync_state.current_cycle_height, Some(72)); - assert_eq!(manager.sync_state.current_cycle_attempts, 0); - assert_eq!(manager.sync_state.last_window_qrinfo_tip, None); - assert_eq!(manager.progress.rotation_cycles(), 2); - manager.sync_state.record_qrinfo_attempt(84); - assert_eq!(manager.sync_state.current_cycle_attempts, 1); - assert_eq!(manager.sync_state.last_window_qrinfo_tip, Some(84)); - assert!(matches!(manager.next_pipeline_mode(84), PipelineMode::Incremental)); - - // If the caller decides not to fire (e.g. another QRInfo is already in - // flight), the per-tip gate stays open so the next call re-picks - // QuorumValidation. The decider must not eagerly burn the gate. - let mut manager = create_test_manager_for(dashcore::Network::Regtest).await; - assert!(matches!(manager.next_pipeline_mode(60), PipelineMode::QuorumValidation { .. })); - assert!(matches!(manager.next_pipeline_mode(60), PipelineMode::QuorumValidation { .. })); - assert_eq!(manager.sync_state.current_cycle_attempts, 0); - assert_eq!(manager.sync_state.last_window_qrinfo_tip, None); - - // Tip before the mining window opens stays Incremental and does not - // touch per-cycle state. - let mut manager = create_test_manager_for(dashcore::Network::Regtest).await; - assert!(matches!(manager.next_pipeline_mode(50), PipelineMode::Incremental)); - assert_eq!(manager.sync_state.current_cycle_height, Some(48)); - assert_eq!(manager.sync_state.current_cycle_attempts, 0); - assert_eq!(manager.sync_state.last_window_qrinfo_tip, None); - - // Tip past the mining window with no prior attempts picks the catch-up - // QuorumValidation. After the caller fires, subsequent calls at any - // tip past the window fall through to Incremental for the rest of the - // cycle. `rotation_cycles` bumps once for entering the cycle and again - // when the tip crosses into the next cycle. - let mut manager = create_test_manager_for(dashcore::Network::Regtest).await; - assert!(matches!(manager.next_pipeline_mode(70), PipelineMode::QuorumValidation { .. })); - assert_eq!(manager.sync_state.current_cycle_height, Some(48)); - assert_eq!(manager.sync_state.current_cycle_attempts, 0); - assert_eq!(manager.progress.rotation_cycles(), 1); - manager.sync_state.record_qrinfo_attempt(70); - assert!(matches!(manager.next_pipeline_mode(70), PipelineMode::Incremental)); - assert!(matches!(manager.next_pipeline_mode(71), PipelineMode::Incremental)); - assert_eq!(manager.progress.rotation_cycles(), 1); - assert!(matches!(manager.next_pipeline_mode(96), PipelineMode::Incremental)); - assert_eq!(manager.sync_state.current_cycle_height, Some(96)); - assert_eq!(manager.progress.rotation_cycles(), 2); - - // `mark_cycle_validated` short-circuits any subsequent tip in that - // cycle to Incremental, even tips inside the mining window. Calling - // it twice for the same cycle bumps `validated_cycles` only once. - let mut manager = create_test_manager_for(dashcore::Network::Regtest).await; - manager.mark_cycle_validated(48); - manager.mark_cycle_validated(48); - assert_eq!(manager.progress.validated_cycles(), 1); - assert!(matches!(manager.next_pipeline_mode(60), PipelineMode::Incremental)); - assert!(matches!(manager.next_pipeline_mode(65), PipelineMode::Incremental)); - assert!(matches!(manager.next_pipeline_mode(50), PipelineMode::Incremental)); - } - - /// On restart, `MasternodesManager::new` must recover - /// `last_synced_block_hash` from the engine's stored masternode lists so - /// the next pipeline run can target the correct base. Without recovery, - /// `send_tip_mnlistdiff_update` would early-return for lack of a base - /// hash and the SPV would re-run the full QRInfo flow on every restart - /// instead of resuming. - #[tokio::test] - async fn test_masternode_manager_recovers_last_synced_hash_from_engine() { - let storage = DiskStorageManager::with_temp_dir().await.unwrap(); - let mut engine = MasternodeListEngine::default_for_network(dashcore::Network::Testnet); - let tip_hash = BlockHash::from_byte_array([0xAB; 32]); - let mid_hash = BlockHash::from_byte_array([0xCD; 32]); - engine.masternode_lists.insert(100, MasternodeList::empty(mid_hash, 100)); - engine.masternode_lists.insert(200, MasternodeList::empty(tip_hash, 200)); - - let manager = MasternodesManager::new( - storage.block_headers(), - Arc::new(RwLock::new(engine)), - dashcore::Network::Testnet, - ) - .await; - - assert_eq!( - manager.sync_state.last_synced_block_hash, - Some(tip_hash), - "new() must recover last_synced_block_hash from the engine's tip list" - ); - assert_eq!( - manager.progress.current_height(), - 200, - "new() must seed progress.current_height from the engine's tip list height" - ); - } - - /// Counterpart to the recovery test: when the engine has no stored - /// masternode lists, `new()` must leave `last_synced_block_hash` as None - /// so the QRInfo path knows it must run from scratch instead of trying - /// to issue a targeted GetMnListDiff against a bogus base. - #[tokio::test] - async fn test_masternode_manager_starts_clean_with_empty_engine() { - let manager = create_test_manager_for(dashcore::Network::Testnet).await; - assert_eq!(manager.sync_state.last_synced_block_hash, None); - assert_eq!(manager.progress.current_height(), 0); - } - - /// `complete_pipeline` after `Incremental` re-evaluates the cycle gate at - /// the latest tip and fires a catch-up QRInfo when the gate picks - /// `QuorumValidation`. When a batch of headers lands while a prior - /// `Incremental` is in flight, every intermediate `BlockHeadersStored` - /// event is rejected by the `has_pending_requests` guard, and the tick - /// handler can't re-fire because `current_height == block_header_tip_height` - /// once the `Incremental` catches up. For DKG_TEST_DIP0024 (regtest), - /// cycle 48 has mining window 60..=68; at tip 70 with no prior attempts, - /// the gate picks catch-up QRInfo. The post-completion call to - /// `next_pipeline_mode` is the first to enter cycle 48 and bumps - /// `rotation_cycles` from 0 to 1. - #[tokio::test] - async fn test_complete_incremental_fires_catch_up_when_window_missed() { - let (mut manager, requests, mut rx) = make_synced_incremental_manager(70).await; - - manager.complete_pipeline(&requests).await.expect("complete_pipeline succeeds"); - - assert_eq!( - manager.sync_state.current_cycle_height, - Some(48), - "post-completion re-eval must call `next_pipeline_mode` and enter cycle 48" - ); - assert_eq!( - manager.progress.rotation_cycles(), - 1, - "entering cycle 48 once via the catch-up branch bumps `rotation_cycles`" - ); - assert_eq!( - manager.progress.qr_infos_requested(), - 1, - "the catch-up branch must reach `send_qrinfo_for_tip` and bump `qr_infos_requested`" - ); - assert!( - manager.sync_state.qrinfo_in_flight.is_some(), - "the catch-up branch must mark a QRInfo as in flight" - ); - let queued = rx.try_recv().expect("a NetworkRequest must be queued by the catch-up"); - assert!( - matches!(queued, NetworkRequest::SendMessage(NetworkMessage::GetQRInfo(_))), - "the queued request must be a `GetQRInfo`, got {:?}", - queued - ); - } - - /// `send_qrinfo_for_tip` must not strand the manager in `Syncing` when - /// the network send fails. A buffered `BlockHeaderSyncComplete` consumed - /// during `WaitingForConnections` reaches `send_qrinfo_for_tip` while no - /// peers are connected. If state transitions before the failing send, - /// `tick` cannot recover because it gates on `qrinfo_in_flight.is_some()`. - #[tokio::test] - async fn test_send_qrinfo_for_tip_preserves_state_when_send_fails() { - let (mut manager, requests, rx) = make_synced_incremental_manager(70).await; - manager.set_state(SyncState::WaitingForConnections); - drop(rx); - - let err = manager - .send_qrinfo_for_tip(&requests) - .await - .expect_err("send must fail when the receiver is dropped"); - assert!(matches!(err, SyncError::Network(_)), "expected Network error, got {:?}", err); - - assert_eq!(manager.state(), SyncState::WaitingForConnections); - assert!(manager.sync_state.qrinfo_in_flight.is_none()); - assert_eq!(manager.progress.qr_infos_requested(), 0); - } - - /// When the cycle gate picks `Incremental` after an `Incremental` - /// completes (e.g. the tip is still before the mining window), the - /// catch-up branch must be a no-op. Cycle 48 mining window is 60..=68 - /// for DKG_TEST_DIP0024; tip 50 is before the window so the gate falls - /// through to `Incremental` and no QRInfo fires. - #[tokio::test] - async fn test_complete_incremental_does_not_fire_when_gate_picks_incremental() { - let (mut manager, requests, _rx) = make_synced_incremental_manager(50).await; - - manager.complete_pipeline(&requests).await.expect("complete_pipeline succeeds"); - - assert!( - manager.sync_state.qrinfo_in_flight.is_none(), - "no QRInfo must fire when the gate picks Incremental" - ); - assert_eq!( - manager.progress.qr_infos_requested(), - 0, - "no QRInfo must be requested when the gate picks Incremental" - ); - } -} diff --git a/dash-spv/src/sync/masternodes/pipeline.rs b/dash-spv/src/sync/masternodes/pipeline.rs index 8c7a0958b..7d21176ce 100644 --- a/dash-spv/src/sync/masternodes/pipeline.rs +++ b/dash-spv/src/sync/masternodes/pipeline.rs @@ -1,30 +1,25 @@ //! MnListDiff pipeline implementation. //! //! Handles pipelined download of MnListDiff messages for quorum validation. -//! Uses DownloadCoordinator for request tracking with timeout and retry logic. +//! Tracks requests with a generic download coordinator. use std::collections::HashMap; -use std::time::Duration; +use std::sync::Arc; use crate::error::SyncResult; -use crate::network::RequestSender; -use crate::sync::download_coordinator::{DownloadConfig, DownloadCoordinator}; -use dashcore::network::message_sml::MnListDiff; +use crate::network2::PeerNetworkManager; +use crate::sync::download_coordinator::DownloadCoordinator; +use dashcore::network::message::NetworkMessage; +use dashcore::network::message_sml::{GetMnListDiff, MnListDiff}; use dashcore::BlockHash; -/// Maximum concurrent MnListDiff requests. -const MAX_CONCURRENT_MNLISTDIFF: usize = 20; - -/// Timeout for MnListDiff requests. -const MNLISTDIFF_TIMEOUT: Duration = Duration::from_secs(15); - /// Pipeline for downloading MnListDiff messages for quorum validation. /// -/// Uses `DownloadCoordinator` for request tracking (keyed by target block_hash), -/// with a HashMap to store the base hash for each request. +/// Tracks requests by target block_hash using a download coordinator, with a +/// HashMap to store the base hash for each request. #[derive(Debug)] pub(super) struct MnListDiffPipeline { - /// Core coordinator tracks requests by target block_hash. + /// Coordinates pending and in-flight requests, keyed by target block_hash. coordinator: DownloadCoordinator, /// Maps target_hash -> base_hash for each request. base_hashes: HashMap, @@ -40,11 +35,7 @@ impl MnListDiffPipeline { /// Create a new MnListDiff pipeline. pub(super) fn new() -> Self { Self { - coordinator: DownloadCoordinator::new( - DownloadConfig::default() - .with_max_concurrent(MAX_CONCURRENT_MNLISTDIFF) - .with_timeout(MNLISTDIFF_TIMEOUT), - ), + coordinator: DownloadCoordinator::new(), base_hashes: HashMap::new(), } } @@ -72,7 +63,10 @@ impl MnListDiffPipeline { /// Send pending requests. /// /// Returns the number of requests sent. - pub(super) fn send_pending(&mut self, requests: &RequestSender) -> SyncResult<()> { + pub(super) async fn send_pending( + &mut self, + network: &Arc, + ) -> SyncResult<()> { let count = self.coordinator.available_to_send(); if count == 0 { return Ok(()); @@ -86,7 +80,12 @@ impl MnListDiffPipeline { continue; }; - requests.request_mnlist_diff(base_hash, target_hash)?; + network + .send(NetworkMessage::GetMnListD(GetMnListDiff { + base_block_hash: base_hash, + block_hash: target_hash, + })) + .await; self.coordinator.mark_sent(&[target_hash]); tracing::debug!( @@ -106,6 +105,11 @@ impl MnListDiffPipeline { self.coordinator.is_in_flight(&diff.block_hash) } + /// Start the response timeout for a request the router just put on the wire. + pub(super) fn mark_on_flight(&mut self, target_hash: &BlockHash) { + self.coordinator.mark_on_flight(&[*target_hash]); + } + /// Receive a MnListDiff response. /// /// Returns true if the diff was expected, false if unexpected. @@ -127,242 +131,8 @@ impl MnListDiffPipeline { true } - /// Requeue a received MnListDiff for retry. - /// - /// Removes from in-flight tracking and pushes back to the front of the - /// pending queue. - pub(super) fn requeue(&mut self, diff: &MnListDiff) { - let target_hash = diff.block_hash; - - // Remove from in-flight - self.coordinator.receive(&target_hash); - - // Re-enqueue for retry - self.coordinator.enqueue_retry(target_hash); - tracing::debug!("Requeued MnListDiff for {} for retry", diff.block_hash); - } - - /// Handle timeouts, re-queuing timed out requests. - pub(super) fn handle_timeouts(&mut self) { - for target_hash in self.coordinator.check_timeouts() { - self.coordinator.enqueue_retry(target_hash); - } - } - /// Check if pipeline has no pending work. pub(super) fn is_complete(&self) -> bool { self.coordinator.is_empty() } - - /// Get the number of in-flight requests. - pub(super) fn active_count(&self) -> usize { - self.coordinator.active_count() - } -} - -#[cfg(test)] -mod tests { - use dashcore::transaction::{OutPoint, Transaction}; - use dashcore::{ScriptBuf, TxIn, TxOut, Witness}; - use dashcore_hashes::Hash; - - use super::*; - - /// Create a minimal MnListDiff for testing. - fn create_test_diff(base_hash: BlockHash, target_hash: BlockHash) -> MnListDiff { - // Create a minimal coinbase transaction - let coinbase_tx = Transaction { - version: 1, - lock_time: 0, - input: vec![TxIn { - previous_output: OutPoint::null(), - script_sig: ScriptBuf::new(), - sequence: 0xffffffff, - witness: Witness::new(), - }], - output: vec![TxOut { - value: 0, - script_pubkey: ScriptBuf::new(), - }], - special_transaction_payload: None, - }; - - MnListDiff { - version: 1, - base_block_hash: base_hash, - block_hash: target_hash, - total_transactions: 1, - merkle_hashes: vec![], - merkle_flags: vec![], - coinbase_tx, - deleted_masternodes: vec![], - new_masternodes: vec![], - deleted_quorums: vec![], - new_quorums: vec![], - quorums_chainlock_signatures: vec![], - } - } - - #[test] - fn test_pipeline_new() { - let pipeline = MnListDiffPipeline::new(); - assert!(pipeline.is_complete()); - assert_eq!(pipeline.active_count(), 0); - } - - #[test] - fn test_queue_requests() { - let mut pipeline = MnListDiffPipeline::new(); - - let base1 = BlockHash::from_byte_array([0x01; 32]); - let target1 = BlockHash::from_byte_array([0x02; 32]); - let base2 = BlockHash::from_byte_array([0x03; 32]); - let target2 = BlockHash::from_byte_array([0x04; 32]); - - pipeline.queue_requests(vec![(base1, target1), (base2, target2)]); - - assert!(!pipeline.is_complete()); - assert_eq!(pipeline.coordinator.pending_count(), 2); - assert_eq!(pipeline.base_hashes.len(), 2); - assert_eq!(pipeline.base_hashes.get(&target1), Some(&base1)); - assert_eq!(pipeline.base_hashes.get(&target2), Some(&base2)); - } - - #[test] - fn test_match_response() { - let mut pipeline = MnListDiffPipeline::new(); - - let base = BlockHash::from_byte_array([0x01; 32]); - let target = BlockHash::from_byte_array([0x02; 32]); - - pipeline.queue_requests(vec![(base, target)]); - - // Take and mark as sent - let items = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&items); - - // Create a test diff - let diff = create_test_diff(base, target); - assert!(pipeline.match_response(&diff)); - - // Unknown hash should not match - let unknown_diff = create_test_diff(base, BlockHash::from_byte_array([0xFF; 32])); - assert!(!pipeline.match_response(&unknown_diff)); - } - - #[test] - fn test_receive() { - let mut pipeline = MnListDiffPipeline::new(); - - let base = BlockHash::from_byte_array([0x01; 32]); - let target = BlockHash::from_byte_array([0x02; 32]); - - pipeline.queue_requests(vec![(base, target)]); - - // Take and mark as sent - let items = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&items); - - let diff = create_test_diff(base, target); - assert!(pipeline.receive(&diff)); - assert!(pipeline.is_complete()); - assert!(pipeline.base_hashes.is_empty()); - } - - #[test] - fn test_receive_unexpected() { - let mut pipeline = MnListDiffPipeline::new(); - - let diff = create_test_diff( - BlockHash::from_byte_array([0x01; 32]), - BlockHash::from_byte_array([0x02; 32]), - ); - - // Receiving unexpected diff should return false - assert!(!pipeline.receive(&diff)); - } - - #[test] - fn test_clear() { - let mut pipeline = MnListDiffPipeline::new(); - - let base = BlockHash::from_byte_array([0x01; 32]); - let target = BlockHash::from_byte_array([0x02; 32]); - - pipeline.queue_requests(vec![(base, target)]); - pipeline.clear(); - - assert!(pipeline.is_complete()); - assert!(pipeline.base_hashes.is_empty()); - } - - #[test] - fn test_handle_timeouts() { - use std::time::Duration; - - let mut pipeline = MnListDiffPipeline { - coordinator: DownloadCoordinator::new( - DownloadConfig::default().with_timeout(Duration::from_millis(1)), - ), - base_hashes: HashMap::new(), - }; - - let base = BlockHash::from_byte_array([0x01; 32]); - let target = BlockHash::from_byte_array([0x02; 32]); - - pipeline.base_hashes.insert(target, base); - pipeline.coordinator.mark_sent(&[target]); - - std::thread::sleep(Duration::from_millis(5)); - - // Timeout re-queues the request, base_hashes preserved - pipeline.handle_timeouts(); - assert_eq!(pipeline.coordinator.pending_count(), 1); - assert!(pipeline.base_hashes.contains_key(&target)); - } - - #[test] - fn test_requeue_puts_back_in_pending() { - let mut pipeline = MnListDiffPipeline::new(); - - let base = BlockHash::from_byte_array([0x01; 32]); - let target = BlockHash::from_byte_array([0x02; 32]); - - pipeline.queue_requests(vec![(base, target)]); - - // Take and mark as sent (simulates sending the request) - let items = pipeline.coordinator.take_pending(1); - pipeline.coordinator.mark_sent(&items); - assert_eq!(pipeline.active_count(), 1); - assert_eq!(pipeline.coordinator.pending_count(), 0); - - let diff = create_test_diff(base, target); - - // Requeue should move from in-flight back to pending - pipeline.requeue(&diff); - assert_eq!(pipeline.active_count(), 0); - assert_eq!(pipeline.coordinator.pending_count(), 1); - // base_hash mapping should be preserved for the retry - assert!(pipeline.base_hashes.contains_key(&target)); - // Pipeline should not be considered complete - assert!(!pipeline.is_complete()); - } - - #[test] - fn test_requeue_always_succeeds() { - let mut pipeline = MnListDiffPipeline::new(); - - let base = BlockHash::from_byte_array([0x01; 32]); - let target = BlockHash::from_byte_array([0x02; 32]); - - pipeline.base_hashes.insert(target, base); - pipeline.coordinator.mark_sent(&[target]); - - let diff = create_test_diff(base, target); - - // Requeue always succeeds - pipeline.requeue(&diff); - assert!(pipeline.base_hashes.contains_key(&target)); - assert_eq!(pipeline.coordinator.pending_count(), 1); - } } diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index e949c0d2d..fbfb9672c 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -1,6 +1,6 @@ use super::manager::PipelineMode; use crate::error::SyncResult; -use crate::network::{Message, MessageType, RequestSender}; +use crate::network2::PeerNetworkManager; use crate::storage::BlockHeaderStorage; use crate::sync::{ ManagerIdentifier, MasternodesManager, SyncEvent, SyncManager, SyncManagerProgress, SyncState, @@ -13,6 +13,8 @@ use dashcore::sml::masternode_list_engine::{MasternodeListEngine, WORK_DIFF_DEPT use dashcore::{BlockHash, QuorumHash}; use dashcore_hashes::Hash; use std::collections::{BTreeSet, HashSet}; +use std::net::SocketAddr; +use std::sync::Arc; use std::time::Duration; /// Per-attempt timeout schedule for QRInfo, indexed by the in-flight attempt's @@ -221,8 +223,15 @@ impl SyncManager for MasternodesManager { self.progress.update_target_height(height); } - fn wanted_message_types(&self) -> &'static [MessageType] { - &[MessageType::MnListDiff, MessageType::QRInfo] + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { + use crate::network2::MessageType; + &[MessageType::MnListDiff, MessageType::QrInfo] + } + + fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { + if let crate::network2::RequestKey::MnListDiff(target_hash) = key { + self.sync_state.mnlistdiff_pipeline.mark_on_flight(target_hash); + } } fn on_disconnect(&mut self) { @@ -233,10 +242,12 @@ impl SyncManager for MasternodesManager { async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult> { - match msg.inner() { + let _ = peer; + match &msg { NetworkMessage::QRInfo(qr_info) => { if !self.sync_state.should_process_qrinfo(qr_info) { return Ok(vec![]); @@ -252,7 +263,7 @@ impl SyncManager for MasternodesManager { tracing::info!("Fed {} block heights to engine", fed); // Feed QRInfo to engine first to populate masternode lists - let qr_info_result = match engine.feed_qr_info(qr_info.clone(), true, true) { + let qr_info_result = match engine.feed_qr_info((**qr_info).clone(), true, true) { Ok(qr_info_result) => qr_info_result, Err(e) => { tracing::error!("QRInfo feed into engine failed: {}", e); @@ -315,13 +326,13 @@ impl SyncManager for MasternodesManager { qr_info_result, }; self.sync_state.mnlistdiff_pipeline.queue_requests(request_pairs); - self.sync_state.mnlistdiff_pipeline.send_pending(requests)?; + self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; self.progress.bump_last_activity(); // If no pending requests, complete if !self.sync_state.has_pending_requests() { - return self.complete_pipeline(requests).await; + return self.complete_pipeline(network).await; } } @@ -341,21 +352,27 @@ impl SyncManager for MasternodesManager { Ok(Some(h)) => h, Ok(None) => { tracing::warn!( - "Height not found for MnListDiff block {}, requeuing for retry", + "Height not found for MnListDiff block {}, dropping", diff.block_hash ); - self.sync_state.mnlistdiff_pipeline.requeue(diff); - self.sync_state.mnlistdiff_pipeline.send_pending(requests)?; + drop(storage); + self.sync_state.mnlistdiff_pipeline.receive(diff); + if self.sync_state.mnlistdiff_pipeline.is_complete() { + return self.complete_pipeline(network).await; + } return Ok(vec![]); } Err(e) => { tracing::warn!( - "Failed to get height for MnListDiff block {}: {}, requeuing for retry", + "Failed to get height for MnListDiff block {}: {}, dropping", diff.block_hash, e ); - self.sync_state.mnlistdiff_pipeline.requeue(diff); - self.sync_state.mnlistdiff_pipeline.send_pending(requests)?; + drop(storage); + self.sync_state.mnlistdiff_pipeline.receive(diff); + if self.sync_state.mnlistdiff_pipeline.is_complete() { + return self.complete_pipeline(network).await; + } return Ok(vec![]); } }; @@ -366,7 +383,7 @@ impl SyncManager for MasternodesManager { engine.feed_block_height(target_height, diff.block_hash); let apply_ok = - match engine.apply_diff(diff.clone(), Some(target_height), false, None) { + match engine.apply_diff((**diff).clone(), Some(target_height), false, None) { Ok(_) => { self.sync_state.known_mn_list_heights.insert(target_height); tracing::debug!("Applied MnListDiff at height {}", target_height); @@ -385,7 +402,7 @@ impl SyncManager for MasternodesManager { self.progress.add_diffs_processed(1); self.sync_state.mnlistdiff_pipeline.receive(diff); - self.sync_state.mnlistdiff_pipeline.send_pending(requests)?; + self.sync_state.mnlistdiff_pipeline.send_pending(network).await?; // Check if all responses received if self.sync_state.mnlistdiff_pipeline.is_complete() { @@ -399,7 +416,7 @@ impl SyncManager for MasternodesManager { return Ok(vec![]); } tracing::info!("All MnListDiff responses received"); - return self.complete_pipeline(requests).await; + return self.complete_pipeline(network).await; } } @@ -412,7 +429,7 @@ impl SyncManager for MasternodesManager { async fn handle_sync_event( &mut self, event: &SyncEvent, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { // Track block header tip height as headers come in if let SyncEvent::BlockHeadersStored { @@ -462,7 +479,7 @@ impl SyncManager for MasternodesManager { ); self.sync_state.qrinfo_retry_count = 0; self.sync_state.clear_pending(); - return self.send_qrinfo_for_tip(requests).await; + return self.send_qrinfo_for_tip(network).await; } PipelineMode::Incremental => { tracing::debug!( @@ -470,7 +487,7 @@ impl SyncManager for MasternodesManager { tip_height, self.progress.current_height() ); - return self.send_tip_mnlistdiff_update(requests).await; + return self.send_tip_mnlistdiff_update(network).await; } } } @@ -544,10 +561,10 @@ impl SyncManager for MasternodesManager { } self.sync_state.qrinfo_retry_count = 0; self.sync_state.clear_pending(); - return self.send_qrinfo_for_tip(requests).await; + return self.send_qrinfo_for_tip(network).await; } PipelineMode::Incremental => { - return self.send_tip_mnlistdiff_update(requests).await; + return self.send_tip_mnlistdiff_update(network).await; } } } @@ -557,14 +574,14 @@ impl SyncManager for MasternodesManager { ); self.sync_state.qrinfo_retry_count = 0; self.sync_state.clear_pending(); - return self.send_qrinfo_for_tip(requests).await; + return self.send_qrinfo_for_tip(network).await; } } Ok(vec![]) } - async fn tick(&mut self, requests: &RequestSender) -> SyncResult> { + async fn tick(&mut self, network: &Arc) -> SyncResult> { // Handle ticks for both Syncing (initial) and Synced (incremental updates) if !matches!(self.state(), SyncState::Syncing | SyncState::Synced) { return Ok(vec![]); @@ -585,11 +602,11 @@ impl SyncManager for MasternodesManager { if self.sync_state.qrinfo_in_flight.is_none() { self.sync_state.qrinfo_retry_count = 0; self.sync_state.clear_pending(); - return self.send_qrinfo_for_tip(requests).await; + return self.send_qrinfo_for_tip(network).await; } } PipelineMode::Incremental => { - return self.send_tip_mnlistdiff_update(requests).await; + return self.send_tip_mnlistdiff_update(network).await; } } } @@ -608,33 +625,19 @@ impl SyncManager for MasternodesManager { ); self.sync_state.qrinfo_retry_count += 1; self.sync_state.clear_pending(); - return self.send_qrinfo_for_tip(requests).await; + return self.send_qrinfo_for_tip(network).await; } else { tracing::warn!( "QRInfo timeout after {} retries, skipping masternode sync", MAX_RETRY_ATTEMPTS ); self.sync_state.clear_pending(); - return self.complete_pipeline(requests).await; + return self.complete_pipeline(network).await; } } return Ok(vec![]); } - // Check for MnListDiff timeouts via pipeline - if self.sync_state.mnlistdiff_pipeline.active_count() > 0 { - self.sync_state.mnlistdiff_pipeline.handle_timeouts(); - - // Send any re-queued requests - self.sync_state.mnlistdiff_pipeline.send_pending(requests)?; - - // Check if complete after handling timeouts - if self.sync_state.mnlistdiff_pipeline.is_complete() { - tracing::info!("MnListDiff pipeline complete"); - return self.complete_pipeline(requests).await; - } - } - Ok(vec![]) } @@ -642,309 +645,3 @@ impl SyncManager for MasternodesManager { SyncManagerProgress::Masternodes(self.progress.clone()) } } - -#[cfg(test)] -mod tests { - use super::super::manager::{MasternodeSyncState, QRInfoInFlight}; - use super::{ - feed_qrinfo_heights_to_engine, qrinfo_timeout_for, MAX_RETRY_ATTEMPTS, - QRINFO_TIMEOUT_SCHEDULE_SECS, - }; - use crate::error::StorageResult; - use crate::storage::{BlockHeaderStorage, BlockHeaderTip}; - use crate::types::HashedBlockHeader; - use async_trait::async_trait; - use dashcore::bls_sig_utils::{BLSPublicKey, BLSSignature}; - use dashcore::hash_types::QuorumVVecHash; - use dashcore::network::message_qrinfo::{MNSkipListMode, QRInfo, QuorumSnapshot}; - use dashcore::network::message_sml::MnListDiff; - use dashcore::sml::llmq_type::LLMQType; - use dashcore::sml::masternode_list_engine::MasternodeListEngine; - use dashcore::transaction::special_transaction::quorum_commitment::QuorumEntry; - use dashcore::{BlockHash, Network, Transaction}; - use dashcore_hashes::Hash; - use std::collections::HashMap; - use std::ops::Range; - use std::time::Instant; - - struct MockHeaderStorage(HashMap); - - #[async_trait] - impl BlockHeaderStorage for MockHeaderStorage { - async fn store_headers(&mut self, _: &[HashedBlockHeader]) -> StorageResult<()> { - Ok(()) - } - async fn store_headers_at_height( - &mut self, - _: &[HashedBlockHeader], - _: u32, - ) -> StorageResult<()> { - Ok(()) - } - async fn load_headers(&self, _: Range) -> StorageResult> { - Ok(vec![]) - } - async fn get_tip_height(&self) -> Option { - None - } - async fn get_tip(&self) -> Option { - None - } - async fn get_start_height(&self) -> Option { - None - } - async fn get_stored_headers_len(&self) -> u32 { - 0 - } - async fn get_header_height_by_hash(&self, hash: &BlockHash) -> StorageResult> { - Ok(self.0.get(hash).copied()) - } - async fn truncate_above(&mut self, target_height: u32) -> StorageResult<()> { - self.0.retain(|_, h| *h <= target_height); - Ok(()) - } - } - - fn make_diff(base_byte: u8, tip_byte: u8) -> MnListDiff { - MnListDiff { - version: 1, - base_block_hash: BlockHash::from_slice(&[base_byte; 32]).unwrap(), - block_hash: BlockHash::from_slice(&[tip_byte; 32]).unwrap(), - total_transactions: 0, - merkle_hashes: vec![], - merkle_flags: vec![], - coinbase_tx: Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }, - deleted_masternodes: vec![], - new_masternodes: vec![], - deleted_quorums: vec![], - new_quorums: vec![], - quorums_chainlock_signatures: vec![], - } - } - - fn make_quorum_entry(hash_byte: u8, index: i16) -> QuorumEntry { - QuorumEntry { - version: 1, - llmq_type: LLMQType::Llmqtype50_60, - quorum_hash: BlockHash::from_slice(&[hash_byte; 32]).unwrap(), - quorum_index: Some(index), - signers: vec![], - valid_members: vec![], - quorum_public_key: BLSPublicKey::from([0u8; 48]), - quorum_vvec_hash: QuorumVVecHash::from_slice(&[0u8; 32]).unwrap(), - threshold_sig: BLSSignature::from([0u8; 96]), - all_commitment_aggregated_signature: BLSSignature::from([0u8; 96]), - } - } - - fn make_snapshot() -> QuorumSnapshot { - QuorumSnapshot { - skip_list_mode: MNSkipListMode::NoSkipping, - active_quorum_members: vec![], - skip_list: vec![], - } - } - - /// Verifies that `feed_qrinfo_heights_to_engine` feeds the engine's - /// `block_container` with heights for every hash source in a `QRInfo` message: - /// - base and tip hashes for each of the five standard diffs - /// - base and tip hashes for the optional h-minus-4c diff - /// - base and tip hashes for each entry in `mn_list_diff_list` - /// - every `QuorumEntry::quorum_hash` in `last_commitment_per_index` - /// - /// The last category is the invariant the parent commit fixed: before that, - /// only Q[0] (the cycle boundary, already present as a diff endpoint) was - /// fed. Q[1]..Q[N-1] were silently missing, causing lookup failures during IS - /// lock and rotated quorum formation verification. - #[tokio::test] - async fn test_feed_qrinfo_heights_to_engine_covers_every_hash_source() { - // Each hash category uses a distinct leading byte so failures are easy to diagnose. - // Diffs: 0x01..0x0E (base/tip pairs for each diff field) - // Commitments: 0x80..0x83 (last_commitment_per_index quorum hashes) - let expected_hashes: &[u8] = &[ - 0x01, 0x02, // mn_list_diff_tip: base, tip - 0x03, 0x04, // mn_list_diff_h: base, tip - 0x05, 0x06, // mn_list_diff_at_h_minus_c: base, tip - 0x07, 0x08, // mn_list_diff_at_h_minus_2c: base, tip - 0x09, 0x0A, // mn_list_diff_at_h_minus_3c: base, tip - 0x0B, 0x0C, // mn_list_diff_at_h_minus_4c: base, tip (optional) - 0x0D, 0x0E, // mn_list_diff_list[0]: base, tip - 0x80, 0x81, 0x82, 0x83, // last_commitment_per_index Q[0]..Q[3] - ]; - - let mut height_map = HashMap::new(); - for (i, &b) in expected_hashes.iter().enumerate() { - height_map.insert(BlockHash::from_slice(&[b; 32]).unwrap(), 100 + i as u32); - } - - let qr_info = QRInfo { - quorum_snapshot_at_h_minus_c: make_snapshot(), - quorum_snapshot_at_h_minus_2c: make_snapshot(), - quorum_snapshot_at_h_minus_3c: make_snapshot(), - mn_list_diff_tip: make_diff(0x01, 0x02), - mn_list_diff_h: make_diff(0x03, 0x04), - mn_list_diff_at_h_minus_c: make_diff(0x05, 0x06), - mn_list_diff_at_h_minus_2c: make_diff(0x07, 0x08), - mn_list_diff_at_h_minus_3c: make_diff(0x09, 0x0A), - quorum_snapshot_and_mn_list_diff_at_h_minus_4c: Some(( - make_snapshot(), - make_diff(0x0B, 0x0C), - )), - mn_list_diff_list: vec![make_diff(0x0D, 0x0E)], - last_commitment_per_index: [0x80u8, 0x81, 0x82, 0x83] - .iter() - .enumerate() - .map(|(i, &b)| make_quorum_entry(b, i as i16)) - .collect(), - quorum_snapshot_list: vec![make_snapshot()], - }; - - let mut engine = MasternodeListEngine { - network: Network::Testnet, - ..Default::default() - }; - feed_qrinfo_heights_to_engine(&mut engine, &qr_info, &MockHeaderStorage(height_map)) - .await - .unwrap(); - - for &b in expected_hashes { - let hash = BlockHash::from_slice(&[b; 32]).unwrap(); - assert!( - engine.block_container.contains_hash(&hash), - "hash 0x{:02X} not fed to engine.block_container", - b - ); - } - } - - /// The QRInfo retry budget escalates: a tight first timeout fails over - /// fast when one peer drops the request silently, while later attempts - /// get progressively more headroom so a genuinely slow but responsive - /// network still gets to answer. - #[test] - fn test_qrinfo_timeout_schedule() { - assert_eq!(QRINFO_TIMEOUT_SCHEDULE_SECS.len(), MAX_RETRY_ATTEMPTS as usize); - - // First attempt fails over fast so a single bad peer does not block sync. - assert_eq!(qrinfo_timeout_for(0).as_secs(), 10); - - // Schedule escalates monotonically so a slow but responsive network - // still gets enough time to answer. - let schedule: Vec = - (0..MAX_RETRY_ATTEMPTS).map(|n| qrinfo_timeout_for(n).as_secs()).collect(); - assert!( - schedule.windows(2).all(|w| w[0] <= w[1]), - "timeout schedule must be non-decreasing, got {:?}", - schedule - ); - - // Out-of-range retry counts must clamp to the slowest slot rather than - // panic, in case the constants drift relative to MAX_RETRY_ATTEMPTS. - let last = *QRINFO_TIMEOUT_SCHEDULE_SECS.last().unwrap(); - assert_eq!(qrinfo_timeout_for(MAX_RETRY_ATTEMPTS).as_secs(), last); - assert_eq!(qrinfo_timeout_for(u8::MAX).as_secs(), last); - } - - /// Build a minimal `QRInfo` whose `mn_list_diff_tip.block_hash` is `[tip_byte; 32]`. - /// Only the tip hash is read by `should_process_qrinfo`; every other field is filler. - fn qrinfo_with_tip(tip_byte: u8) -> QRInfo { - QRInfo { - quorum_snapshot_at_h_minus_c: make_snapshot(), - quorum_snapshot_at_h_minus_2c: make_snapshot(), - quorum_snapshot_at_h_minus_3c: make_snapshot(), - mn_list_diff_tip: make_diff(0x00, tip_byte), - mn_list_diff_h: make_diff(0x00, 0x00), - mn_list_diff_at_h_minus_c: make_diff(0x00, 0x00), - mn_list_diff_at_h_minus_2c: make_diff(0x00, 0x00), - mn_list_diff_at_h_minus_3c: make_diff(0x00, 0x00), - quorum_snapshot_and_mn_list_diff_at_h_minus_4c: None, - mn_list_diff_list: vec![], - last_commitment_per_index: vec![], - quorum_snapshot_list: vec![], - } - } - - /// `should_process_qrinfo` is the dedup gate at the QRInfo handler entry. It - /// must: - /// 1. Drop a response carrying the same `mn_list_diff_tip.block_hash` as the - /// last successfully processed one (defends against a late straggler from - /// a previous request whose response already won, even when the in-flight - /// gate is open for a newer request). - /// 2. Drop an unsolicited response (no QRInfo currently in flight). - /// 3. Allow a fresh response that matches the active in-flight request tip. - /// 4. Drop a response whose tip does not match the active in-flight request - /// tip (late straggler from a previous tip whose request was rotated by a - /// timeout retry). - #[test] - fn test_should_process_qrinfo_dedup_gate() { - let tip_a = BlockHash::from_slice(&[0xAA; 32]).unwrap(); - let tip_b = BlockHash::from_slice(&[0xBB; 32]).unwrap(); - let in_flight_b = QRInfoInFlight { - tip: tip_b, - wait_start: Instant::now(), - }; - - // Same-tip duplicate is dropped even when a request is in flight. - let state = MasternodeSyncState { - qrinfo_in_flight: Some(in_flight_b), - last_processed_qrinfo_tip: Some(tip_a), - ..Default::default() - }; - assert!( - !state.should_process_qrinfo(&qrinfo_with_tip(0xAA)), - "duplicate of last processed tip must be dropped" - ); - - // Unsolicited response (no request in flight) is dropped, even for a - // fresh tip. - let state = MasternodeSyncState::default(); - assert!(state.qrinfo_in_flight.is_none()); - assert!( - !state.should_process_qrinfo(&qrinfo_with_tip(0xBB)), - "unsolicited response must be dropped" - ); - - // Response matching the active in-flight tip is accepted. - let state = MasternodeSyncState { - qrinfo_in_flight: Some(in_flight_b), - last_processed_qrinfo_tip: Some(tip_a), - ..Default::default() - }; - assert!( - state.should_process_qrinfo(&qrinfo_with_tip(0xBB)), - "response matching the active request tip must be accepted" - ); - - // Same-tip dedup wins over the in-flight check: even if the flag has - // already been cleared (e.g. a sibling response just flipped it), the - // straggler must not be processed twice. - let state = MasternodeSyncState { - qrinfo_in_flight: None, - last_processed_qrinfo_tip: Some(tip_a), - ..Default::default() - }; - assert!( - !state.should_process_qrinfo(&qrinfo_with_tip(0xAA)), - "duplicate must be dropped even when no request is in flight" - ); - - // Late straggler from a previous tip whose request was rotated by a - // timeout retry: the in-flight gate is open for tip B, but tip C - // arrives. Dropped because the tip does not match the active request. - let state = MasternodeSyncState { - qrinfo_in_flight: Some(in_flight_b), - last_processed_qrinfo_tip: Some(tip_a), - ..Default::default() - }; - assert!( - !state.should_process_qrinfo(&qrinfo_with_tip(0xCC)), - "response for non-active request tip must be dropped" - ); - } -} diff --git a/dash-spv/src/sync/mempool/manager.rs b/dash-spv/src/sync/mempool/manager.rs index fc8fd23ea..3741acbce 100644 --- a/dash-spv/src/sync/mempool/manager.rs +++ b/dash-spv/src/sync/mempool/manager.rs @@ -21,7 +21,7 @@ use super::filter::build_wallet_bloom_filter; use super::BLOOM_FALSE_POSITIVE_RATE; use crate::client::config::MempoolStrategy; use crate::error::SyncResult; -use crate::network::RequestSender; +use crate::network2::PeerNetworkManager; use crate::sync::mempool::MempoolProgress; use crate::sync::SyncEvent; use crate::types::UnconfirmedTransaction; @@ -106,30 +106,33 @@ impl MempoolManager { pub(super) async fn activate_peer( &mut self, peer: SocketAddr, - requests: &RequestSender, + network: &Arc, ) -> SyncResult<()> { tracing::info!("Activating mempool on peer {} (strategy: {:?})", peer, self.strategy); match self.strategy { MempoolStrategy::BloomFilter => { - self.load_bloom_filter(peer, requests).await?; + self.load_bloom_filter(peer, network).await?; } MempoolStrategy::FetchAll => { - requests.send_filter_clear(peer)?; + network.send(NetworkMessage::FilterClear).await; } } - requests.request_mempool(peer)?; + network.send(NetworkMessage::MemPool).await; self.peers.insert(peer, Some(VecDeque::new())); Ok(()) } /// Activate mempool relay on all connected but not-yet-activated peers. - pub(super) async fn activate_all_peers(&mut self, requests: &RequestSender) -> SyncResult<()> { + pub(super) async fn activate_all_peers( + &mut self, + network: &Arc, + ) -> SyncResult<()> { let inactive: Vec = self.peers.iter().filter(|(_, v)| v.is_none()).map(|(k, _)| *k).collect(); for peer in inactive { - self.activate_peer(peer, requests).await?; + self.activate_peer(peer, network).await?; } Ok(()) } @@ -138,7 +141,7 @@ impl MempoolManager { async fn load_bloom_filter( &mut self, peer: SocketAddr, - requests: &RequestSender, + network: &Arc, ) -> SyncResult<()> { let wallet = self.wallet.read().await; let addresses = wallet.monitored_addresses(); @@ -165,13 +168,16 @@ impl MempoolManager { filter_load.filter.len() ); - requests.send_filter_load(filter_load, peer)?; + network.send(NetworkMessage::FilterLoad(filter_load)).await; Ok(()) } /// Rebuild the bloom filter on all activated peers. - pub(super) async fn rebuild_filter(&mut self, requests: &RequestSender) -> SyncResult<()> { + pub(super) async fn rebuild_filter( + &mut self, + network: &Arc, + ) -> SyncResult<()> { if self.strategy != MempoolStrategy::BloomFilter { return Ok(()); } @@ -184,9 +190,9 @@ impl MempoolManager { } for peer in activated { - requests.send_filter_clear(peer)?; - self.load_bloom_filter(peer, requests).await?; - requests.request_mempool(peer)?; + network.send(NetworkMessage::FilterClear).await; + self.load_bloom_filter(peer, network).await?; + network.send(NetworkMessage::MemPool).await; } Ok(()) @@ -200,7 +206,7 @@ impl MempoolManager { &mut self, inv: &[Inventory], peer: SocketAddr, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { let mempool_full = self.transactions.len() >= self.max_transactions; if mempool_full { @@ -234,7 +240,7 @@ impl MempoolManager { if enqueued > 0 { tracing::debug!("Enqueued {} mempool txids for download", enqueued); - self.send_queued(requests).await?; + self.send_queued(network).await?; } Ok(vec![]) @@ -244,7 +250,10 @@ impl MempoolManager { /// /// Deduplicates at send time against `pending_requests` and `mempool_state` /// in case a transaction was received between enqueue and send. - pub(super) async fn send_queued(&mut self, requests: &RequestSender) -> SyncResult<()> { + pub(super) async fn send_queued( + &mut self, + network: &Arc, + ) -> SyncResult<()> { let mut available = MAX_IN_FLIGHT.saturating_sub(self.pending_requests.len()); let has_queued = self.peers.values().any(|v| v.as_ref().is_some_and(|q| !q.is_empty())); if available == 0 || !has_queued { @@ -294,7 +303,7 @@ impl MempoolManager { peer, total_queued, ); - requests.request_inventory(inventory, peer)?; + network.send(NetworkMessage::GetData(inventory)).await; } Ok(()) } @@ -445,21 +454,21 @@ impl MempoolManager { /// Each transaction in `recent_sends` tracks when it was last broadcast. /// Transactions whose last broadcast was more than `REBROADCAST_INTERVAL` /// ago are rebroadcast and their timestamp is reset. - pub(super) async fn rebroadcast_if_due(&mut self, requests: &RequestSender) { - self.rebroadcast_if_due_at(requests, Instant::now()).await + pub(super) async fn rebroadcast_if_due(&mut self, network: &Arc) { + self.rebroadcast_if_due_at(network, Instant::now()).await } /// `now`-injected variant of [`Self::rebroadcast_if_due`]. Tests project `now` /// forward instead of subtracting from `Instant::now()`, which underflows on /// Windows when the QPC-based monotonic clock has a small value at boot. - async fn rebroadcast_if_due_at(&mut self, requests: &RequestSender, now: Instant) { + async fn rebroadcast_if_due_at(&mut self, network: &Arc, now: Instant) { let mut count: usize = 0; for (txid, last_broadcast) in &mut self.recent_sends { if now.saturating_duration_since(*last_broadcast) < REBROADCAST_INTERVAL { continue; } if let Some(unconfirmed) = self.transactions.get(txid) { - let _ = requests.broadcast(NetworkMessage::Tx(unconfirmed.transaction.clone())); + network.broadcast(NetworkMessage::Tx(Box::new(unconfirmed.transaction.clone()))); *last_broadcast = now; count += 1; } @@ -554,1167 +563,3 @@ impl fmt::Debug for MempoolManager { .finish() } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::network::NetworkRequest; - use dashcore::hashes::Hash; - use dashcore::network::message::NetworkMessage; - use dashcore::{Address, BlockHash, Network, ScriptBuf, Transaction}; - use key_wallet::transaction_checking::TransactionContext; - use key_wallet_manager::test_utils::MockWallet; - - use crate::sync::SyncState; - use crate::test_utils::test_socket_address; - use tokio::sync::mpsc; - - fn dummy_instant_lock(txid: Txid) -> InstantLock { - InstantLock { - txid, - ..InstantLock::default() - } - } - - fn rich_instant_lock(txid: Txid) -> InstantLock { - InstantLock { - txid, - cyclehash: BlockHash::from_byte_array([0xab; 32]), - ..InstantLock::default() - } - } - - fn create_test_manager( - ) -> (MempoolManager, RequestSender, mpsc::UnboundedReceiver) { - let wallet = Arc::new(RwLock::new(MockWallet::new())); - let (tx, rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new(wallet, MempoolStrategy::FetchAll, 1000, 0); - manager.progress.set_state(SyncState::Synced); - - (manager, requests, rx) - } - - fn create_bloom_manager( - ) -> (MempoolManager, RequestSender, mpsc::UnboundedReceiver) { - let wallet = Arc::new(RwLock::new(MockWallet::new())); - let (tx, rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let manager = MempoolManager::new(wallet, MempoolStrategy::BloomFilter, 1000, 0); - - (manager, requests, rx) - } - - #[tokio::test] - async fn test_activation_fetch_all() { - let peer = test_socket_address(1); - let (mut manager, requests, mut rx) = create_test_manager(); - manager.activate_peer(peer, &requests).await.unwrap(); - - // FetchAll activation sends filterclear then mempool to the chosen peer - let msg1 = rx.recv().await.unwrap(); - assert!( - matches!(msg1, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterClear, p) if p == peer) - ); - let msg2 = rx.recv().await.unwrap(); - assert!( - matches!(msg2, NetworkRequest::SendMessageToPeer(NetworkMessage::MemPool, p) if p == peer) - ); - assert!(matches!(manager.peers.get(&peer), Some(Some(_)))); - } - - #[tokio::test] - async fn test_activation_bloom_filter_skips_empty_wallet() { - let (mut manager, requests, mut rx) = create_bloom_manager(); - manager.activate_peer(test_socket_address(1), &requests).await.unwrap(); - - // No addresses in mock wallet, so only MemPool should be sent (no FilterLoad) - let mut found_filter_load = false; - while let Ok(msg) = rx.try_recv() { - if matches!(msg, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _)) { - found_filter_load = true; - } - } - assert!(!found_filter_load, "should not send FilterLoad for empty wallet"); - } - - #[tokio::test] - async fn test_handle_inv_deduplication() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - let txid = Txid::from_byte_array([1u8; 32]); - let inv = vec![Inventory::Transaction(txid)]; - - // First call should add to pending - let events = manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert!(events.is_empty()); - assert!(manager.pending_requests.contains_key(&txid)); - - // Second call with same txid should be filtered out - let events = manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert!(events.is_empty()); - assert_eq!(manager.pending_requests.len(), 1); - } - - #[tokio::test] - async fn test_handle_inv_capacity_limit() { - let wallet = Arc::new(RwLock::new(MockWallet::new())); - let (tx, _rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new( - wallet, - MempoolStrategy::FetchAll, - 2, // Very small capacity - 0, - ); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - // Fill mempool to capacity - for i in 0..2u32 { - let tx = Transaction { - version: 1, - lock_time: i, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - manager.transactions.insert( - txid, - UnconfirmedTransaction::new(tx, Amount::from_sat(0), false, false, Vec::new(), 0), - ); - } - - // New transactions should be filtered out - let new_txid = Txid::from_byte_array([99u8; 32]); - let inv = vec![Inventory::Transaction(new_txid)]; - let events = manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert!(events.is_empty()); - assert!(!manager.pending_requests.contains_key(&new_txid)); - } - - #[tokio::test] - async fn test_handle_inv_pending_requests_limit() { - let wallet = Arc::new(RwLock::new(MockWallet::new())); - let (tx, _rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new(wallet, MempoolStrategy::FetchAll, 2, 0); - manager.progress.set_state(SyncState::Synced); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - // Fill pending requests to capacity - let inv1: Vec = - (0..2).map(|i| Inventory::Transaction(Txid::from_byte_array([i; 32]))).collect(); - manager.handle_inv(&inv1, peer, &requests).await.unwrap(); - assert_eq!(manager.pending_requests.len(), 2); - - // Additional requests should be rejected when pending is at capacity - let extra_txid = Txid::from_byte_array([99; 32]); - let inv2 = vec![Inventory::Transaction(extra_txid)]; - manager.handle_inv(&inv2, peer, &requests).await.unwrap(); - assert!(!manager.pending_requests.contains_key(&extra_txid)); - } - - #[test] - fn test_prune_pending_requests_timeout() { - let wallet = Arc::new(RwLock::new(MockWallet::new())); - let (tx, _rx) = mpsc::unbounded_channel::(); - let _requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new(wallet, MempoolStrategy::FetchAll, 1000, 0); - - let fresh_txid = Txid::from_byte_array([1; 32]); - let stale_txid = Txid::from_byte_array([2; 32]); - - manager.pending_requests.insert(fresh_txid, Instant::now()); - manager - .pending_requests - .insert(stale_txid, Instant::now() - PENDING_REQUEST_TIMEOUT - Duration::from_secs(1)); - - manager.prune_pending_requests(); - - assert!(manager.pending_requests.contains_key(&fresh_txid)); - assert!(!manager.pending_requests.contains_key(&stale_txid)); - } - - #[tokio::test] - async fn test_handle_tx_irrelevant() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let tx = Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - let events = manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - // MockWallet returns is_relevant=false by default - assert!(events.is_empty()); - assert_eq!(manager.progress.received(), 1); - - // Irrelevant tx should not be stored - assert!(!manager.transactions.contains_key(&txid)); - assert_eq!(manager.progress.relevant(), 0); - } - - #[tokio::test] - async fn test_handle_inv_non_transaction_filtered() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - let inv = vec![ - Inventory::Block(BlockHash::all_zeros()), - Inventory::Transaction(Txid::from_byte_array([1u8; 32])), - ]; - - let events = manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert!(events.is_empty()); - // Only the transaction should be tracked, not the block - assert_eq!(manager.pending_requests.len(), 1); - } - - #[test] - fn test_prune_expired() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let fresh_tx = Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let fresh_txid = fresh_tx.txid(); - - let expired_tx = Transaction { - version: 1, - lock_time: 99, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let expired_txid = expired_tx.txid(); - let test_timeout = Duration::from_secs(2); - - manager.transactions.insert( - fresh_txid, - UnconfirmedTransaction::new(fresh_tx, Amount::from_sat(0), false, false, Vec::new(), 0), - ); - let mut expired_utx = UnconfirmedTransaction::new( - expired_tx, - Amount::from_sat(0), - false, - false, - Vec::new(), - 0, - ); - expired_utx.first_seen = Instant::now() - test_timeout - Duration::from_secs(1); - manager.transactions.insert(expired_txid, expired_utx); - - manager.prune_expired(test_timeout); - - assert_eq!(manager.transactions.len(), 1); - assert!(manager.transactions.contains_key(&fresh_txid)); - assert!(!manager.transactions.contains_key(&expired_txid)); - assert_eq!(manager.progress.removed(), 1); - } - - /// Create a manager with BloomFilter strategy where the wallet reports - /// mempool transactions as relevant. BloomFilter strategy skips local - /// address pre-filtering, relying on the wallet for definitive checks. - fn create_relevant_manager( - ) -> (MempoolManager, RequestSender, Arc>) { - let mut mock = MockWallet::new(); - mock.set_mempool_relevant(true); - let wallet = Arc::new(RwLock::new(mock)); - let (tx, _rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let manager = MempoolManager::new(wallet.clone(), MempoolStrategy::BloomFilter, 1000, 0); - - (manager, requests, wallet) - } - - #[tokio::test] - async fn test_handle_tx_relevant_stores_transaction() { - let (mut manager, _requests, _wallet) = create_relevant_manager(); - - let tx = Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - let events = manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - assert!(events.is_empty()); - - // Verify transaction was stored - assert!(manager.transactions.contains_key(&txid)); - assert_eq!(manager.progress.received(), 1); - assert_eq!(manager.progress.relevant(), 1); - assert_eq!(manager.progress.tracked(), 1); - - // Processing the same transaction again should be a no-op (dedup guard) - let tx2 = Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let events = manager.handle_tx(tx2, test_socket_address(1)).await.unwrap(); - assert!(events.is_empty()); - - assert_eq!(manager.transactions.len(), 1); - // Progress counters should not have incremented - assert_eq!(manager.progress.received(), 1); - assert_eq!(manager.progress.relevant(), 1); - } - - #[tokio::test] - async fn test_handle_tx_local_records_send() { - let (mut manager, _requests, _wallet) = create_relevant_manager(); - - let tx = Transaction { - version: 2, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - // Use the unspecified address to simulate a locally broadcast transaction - let local_addr = SocketAddr::from(([0, 0, 0, 0], 0)); - manager.handle_tx(tx, local_addr).await.unwrap(); - - assert!(manager.transactions.contains_key(&txid)); - assert!( - manager.recent_sends.contains_key(&txid), - "locally dispatched transaction should be recorded as a recent send" - ); - } - - #[tokio::test] - async fn test_handle_tx_remote_does_not_record_send() { - let (mut manager, _requests, _wallet) = create_relevant_manager(); - - let tx = Transaction { - version: 3, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - - assert!(manager.transactions.contains_key(&txid)); - assert!( - !manager.recent_sends.contains_key(&txid), - "peer-received transaction should not be recorded as a recent send" - ); - } - - #[tokio::test] - async fn test_handle_tx_clears_pending_request() { - let (mut manager, _requests, _wallet) = create_relevant_manager(); - - let tx = Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - // Simulate that we requested this transaction - manager.pending_requests.insert(txid, Instant::now()); - assert!(manager.pending_requests.contains_key(&txid)); - - manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - // Pending request should be cleared regardless of relevance - assert!(!manager.pending_requests.contains_key(&txid)); - - // Since the manager uses BloomFilter strategy (relevant mock), tx should be stored - assert!(manager.transactions.contains_key(&txid)); - } - - fn create_bloom_manager_with_addresses( - addresses: Vec
, - ) -> (MempoolManager, RequestSender, mpsc::UnboundedReceiver) { - let mut mock = MockWallet::new(); - mock.set_addresses(addresses); - let wallet = Arc::new(RwLock::new(mock)); - let (tx, rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let manager = MempoolManager::new(wallet, MempoolStrategy::BloomFilter, 1000, 0); - - (manager, requests, rx) - } - - /// Create a test P2PKH address from a byte pattern. - fn test_address(byte: u8) -> Address { - // Build OP_DUP OP_HASH160 <20-byte-hash> OP_EQUALVERIFY OP_CHECKSIG - let mut script_bytes = vec![0x76, 0xa9, 0x14]; // OP_DUP OP_HASH160 PUSH20 - script_bytes.extend_from_slice(&[byte; 20]); - script_bytes.push(0x88); // OP_EQUALVERIFY - script_bytes.push(0xac); // OP_CHECKSIG - let script = ScriptBuf::from(script_bytes); - Address::from_script(&script, Network::Testnet).unwrap() - } - - #[tokio::test] - async fn test_bloom_filter_loaded_with_addresses() { - let addr = test_address(0xab); - - let (mut manager, requests, mut rx) = create_bloom_manager_with_addresses(vec![addr]); - manager.activate_peer(test_socket_address(1), &requests).await.unwrap(); - - let mut found_filter_load = false; - while let Ok(msg) = rx.try_recv() { - if matches!(msg, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _)) { - found_filter_load = true; - } - } - assert!(found_filter_load, "expected FilterLoad for wallet with addresses"); - } - - #[tokio::test] - async fn test_mark_instant_send_emits_status_change() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let tx = Transaction { - version: 1, - lock_time: 42, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - manager.transactions.insert( - txid, - UnconfirmedTransaction::new(tx, Amount::from_sat(0), false, false, Vec::new(), 0), - ); - manager.recent_sends.insert(txid, Instant::now()); - - manager.process_instant_send(dummy_instant_lock(txid)).await; - - // Verify IS flag and recent_sends cleanup - assert!(manager.transactions.get(&txid).unwrap().is_instant_send); - assert!( - !manager.recent_sends.contains_key(&txid), - "IS-locked transaction should be removed from recent_sends" - ); - - let wallet = manager.wallet.read().await; - let status_changes = wallet.status_changes(); - let changes = status_changes.lock().await; - assert_eq!(changes.len(), 1); - assert_eq!(changes[0].0, txid); - assert!(matches!(changes[0].1, TransactionContext::InstantSend(_))); - } - - #[tokio::test] - async fn test_mark_instant_send_stores_pending_for_unknown() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let unknown_txid = Txid::from_byte_array([0xbb; 32]); - manager.process_instant_send(dummy_instant_lock(unknown_txid)).await; - - // No immediate wallet notification - let wallet = manager.wallet.read().await; - let status_changes = wallet.status_changes(); - let changes = status_changes.lock().await; - assert!(changes.is_empty()); - - // But the txid is remembered for when the transaction arrives - assert!(manager.pending_is_locks.contains_key(&unknown_txid)); - } - - #[tokio::test] - async fn test_in_flight_limit() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - // Send 200 INVs — only MAX_IN_FLIGHT should go to pending, rest queued - let inv: Vec = (0..200u16) - .map(|i| { - let mut bytes = [0u8; 32]; - bytes[0..2].copy_from_slice(&i.to_le_bytes()); - Inventory::Transaction(Txid::from_byte_array(bytes)) - }) - .collect(); - - manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert_eq!(manager.pending_requests.len(), MAX_IN_FLIGHT); - assert_eq!( - manager.peers.values().filter_map(|v| v.as_ref()).map(|q| q.len()).sum::(), - 100 - ); - } - - #[tokio::test] - async fn test_send_queued_drains_after_response() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - // Fill with 150 INVs - let inv: Vec = (0..150u16) - .map(|i| { - let mut bytes = [0u8; 32]; - bytes[0..2].copy_from_slice(&i.to_le_bytes()); - Inventory::Transaction(Txid::from_byte_array(bytes)) - }) - .collect(); - - manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert_eq!(manager.pending_requests.len(), MAX_IN_FLIGHT); - assert_eq!( - manager.peers.values().filter_map(|v| v.as_ref()).map(|q| q.len()).sum::(), - 50 - ); - - // Simulate receiving 10 responses (freeing 10 slots) - let pending_txids: Vec = manager.pending_requests.keys().take(10).copied().collect(); - for txid in &pending_txids { - manager.pending_requests.remove(txid); - } - assert_eq!(manager.pending_requests.len(), 90); - - // send_queued should fill the freed slots - manager.send_queued(&requests).await.unwrap(); - assert_eq!(manager.pending_requests.len(), MAX_IN_FLIGHT); - assert_eq!( - manager.peers.values().filter_map(|v| v.as_ref()).map(|q| q.len()).sum::(), - 40 - ); - } - - #[tokio::test] - async fn test_send_queued_skips_already_received() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - - // Create a real transaction and get its actual txid - let tx = Transaction { - version: 1, - lock_time: 0xaa, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - // Enqueue the txid on an activated peer - manager.peers.insert(peer, Some(VecDeque::from([txid]))); - - // Simulate the transaction arriving before send - manager.transactions.insert( - txid, - UnconfirmedTransaction::new(tx, Amount::from_sat(0), false, false, Vec::new(), 0), - ); - - manager.send_queued(&requests).await.unwrap(); - // Txid should have been skipped, not added to pending - assert!(manager.pending_requests.is_empty()); - assert!(manager.peers.values().filter_map(|v| v.as_ref()).all(|q| q.is_empty())); - } - - #[test] - fn test_clear_pending_clears_queue() { - let (mut manager, _requests, _rx) = create_test_manager(); - - manager.pending_requests.insert(Txid::from_byte_array([1; 32]), Instant::now()); - manager - .peers - .insert(test_socket_address(1), Some(VecDeque::from([Txid::from_byte_array([2; 32])]))); - let txid3 = Txid::from_byte_array([3; 32]); - manager.pending_is_locks.insert(txid3, (dummy_instant_lock(txid3), Instant::now())); - - manager.clear_pending(); - - assert!(manager.pending_requests.is_empty()); - assert!(manager.peers.is_empty()); - assert!(manager.pending_is_locks.is_empty()); - } - - #[tokio::test] - async fn test_send_queued_noop_at_capacity() { - let (mut manager, requests, _rx) = create_test_manager(); - - // Fill pending to MAX_IN_FLIGHT - for i in 0..MAX_IN_FLIGHT as u16 { - let mut bytes = [0u8; 32]; - bytes[0..2].copy_from_slice(&i.to_le_bytes()); - manager.pending_requests.insert(Txid::from_byte_array(bytes), Instant::now()); - } - - // Add something to the queue on an activated peer - manager.peers.insert( - test_socket_address(1), - Some(VecDeque::from([Txid::from_byte_array([0xff; 32])])), - ); - - manager.send_queued(&requests).await.unwrap(); - // Queue should remain unchanged (one peer with one txid) - assert_eq!( - manager.peers.values().filter_map(|v| v.as_ref()).map(|q| q.len()).sum::(), - 1 - ); - assert_eq!(manager.pending_requests.len(), MAX_IN_FLIGHT); - } - - #[tokio::test] - async fn test_instant_send_before_transaction() { - let (mut manager, _requests, wallet) = create_relevant_manager(); - - let tx = Transaction { - version: 1, - lock_time: 77, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - // IS lock arrives before the transaction (with a distinct cyclehash) - manager.process_instant_send(rich_instant_lock(txid)).await; - assert!(manager.pending_is_locks.contains_key(&txid)); - - // Transaction arrives - manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - - // Pending IS lock consumed - assert!(manager.pending_is_locks.is_empty()); - - // Transaction stored with IS flag set - assert!(manager.transactions.get(&txid).unwrap().is_instant_send); - - // Wallet received the IS lock payload with the correct cyclehash - let w = wallet.read().await; - let locks = w.processed_instant_locks.lock().await; - let received = locks.iter().find(|(id, lock)| { - *id == txid - && lock - .as_ref() - .is_some_and(|l| l.cyclehash == BlockHash::from_byte_array([0xab; 32])) - }); - assert!(received.is_some(), "wallet should have received rich IS lock with cyclehash 0xab"); - } - - #[tokio::test] - async fn test_instant_send_before_irrelevant_transaction() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let tx = Transaction { - version: 1, - lock_time: 88, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - // IS lock arrives before the transaction - manager.process_instant_send(dummy_instant_lock(txid)).await; - assert!(manager.pending_is_locks.contains_key(&txid)); - - // Transaction arrives but wallet says it's not relevant - manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - - // Pending IS lock cleaned up (no leak) - assert!(manager.pending_is_locks.is_empty()); - - // Irrelevant tx should not be stored - assert!(!manager.transactions.contains_key(&txid)); - } - - #[tokio::test] - async fn test_pending_is_locks_capacity_limit() { - let (mut manager, _requests, _rx) = create_test_manager(); - - // Fill pending IS locks to capacity - for i in 0..MAX_PENDING_IS_LOCKS { - let mut bytes = [0u8; 32]; - bytes[0..8].copy_from_slice(&(i as u64).to_le_bytes()); - let txid = Txid::from_byte_array(bytes); - manager.pending_is_locks.insert(txid, (dummy_instant_lock(txid), Instant::now())); - } - assert_eq!(manager.pending_is_locks.len(), MAX_PENDING_IS_LOCKS); - - // Next IS lock should be dropped - let overflow_txid = Txid::from_byte_array([0xff; 32]); - manager.process_instant_send(dummy_instant_lock(overflow_txid)).await; - assert!(!manager.pending_is_locks.contains_key(&overflow_txid)); - assert_eq!(manager.pending_is_locks.len(), MAX_PENDING_IS_LOCKS); - } - - #[test] - fn test_prune_expired_removes_is_lock_for_expired_tx() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let tx = Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - let test_timeout = Duration::from_secs(2); - - // Add the tx with a timestamp in the past so it expires - let mut utx = - UnconfirmedTransaction::new(tx, Amount::from_sat(0), false, false, Vec::new(), 0); - utx.first_seen = Instant::now() - test_timeout - Duration::from_secs(1); - manager.transactions.insert(txid, utx); - - // Also store a pending IS lock for this txid and an unrelated one - let unrelated_txid = Txid::from_byte_array([0xdd; 32]); - manager.pending_is_locks.insert(txid, (dummy_instant_lock(txid), Instant::now())); - manager - .pending_is_locks - .insert(unrelated_txid, (dummy_instant_lock(unrelated_txid), Instant::now())); - - manager.prune_expired(test_timeout); - - // The expired tx's IS lock should be removed - assert!( - !manager.pending_is_locks.contains_key(&txid), - "IS lock for expired tx should be removed" - ); - // The unrelated IS lock should be preserved - assert!( - manager.pending_is_locks.contains_key(&unrelated_txid), - "IS lock for non-expired tx should be preserved" - ); - } - - #[test] - fn test_prune_expired_removes_stale_pending_is_locks() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let test_timeout = Duration::from_secs(2); - - // Insert a pending IS lock that is older than the test timeout - let stale_txid = Txid::from_byte_array([0xaa; 32]); - manager.pending_is_locks.insert( - stale_txid, - ( - dummy_instant_lock(stale_txid), - Instant::now() - test_timeout - Duration::from_secs(1), - ), - ); - - // Insert a fresh pending IS lock - let fresh_txid = Txid::from_byte_array([0xbb; 32]); - manager - .pending_is_locks - .insert(fresh_txid, (dummy_instant_lock(fresh_txid), Instant::now())); - - manager.prune_expired(test_timeout); - - assert!( - !manager.pending_is_locks.contains_key(&stale_txid), - "stale pending IS lock should be pruned" - ); - assert!( - manager.pending_is_locks.contains_key(&fresh_txid), - "fresh pending IS lock should be preserved" - ); - } - - #[tokio::test] - async fn test_handle_inv_dedup_against_queue() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - // Fill pending to capacity so items go to queue - for i in 0..MAX_IN_FLIGHT as u16 { - let mut bytes = [0u8; 32]; - bytes[0..2].copy_from_slice(&i.to_le_bytes()); - manager.pending_requests.insert(Txid::from_byte_array(bytes), Instant::now()); - } - - let txid = Txid::from_byte_array([0xff; 32]); - let inv = vec![Inventory::Transaction(txid)]; - - // First call enqueues - manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert_eq!( - manager.peers.values().filter_map(|v| v.as_ref()).map(|q| q.len()).sum::(), - 1 - ); - - // Second call with same txid should be deduped - manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert_eq!( - manager.peers.values().filter_map(|v| v.as_ref()).map(|q| q.len()).sum::(), - 1 - ); - } - - #[tokio::test] - async fn test_bloom_filter_load_failure_propagates() { - let addr = test_address(0xab); - let mut mock = MockWallet::new(); - mock.set_addresses(vec![addr]); - let wallet = Arc::new(RwLock::new(mock)); - let (tx, rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new(wallet, MempoolStrategy::BloomFilter, 1000, 0); - - // Drop receiver so send_filter_load fails - drop(rx); - - // activate() should propagate the error - let result = manager.activate_peer(test_socket_address(1), &requests).await; - assert!(result.is_err()); - } - - #[tokio::test] - async fn test_handle_tx_relevant_populates_wallet_effect_fields() { - let (mut manager, _requests, wallet) = create_relevant_manager(); - - let tx = Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - let addr: Address = - "yWdXnYxGbouNoo8yMvcbZmZ3Gdp6BpySxL".parse::>().unwrap().assume_checked(); - { - let mut w = wallet.write().await; - w.set_mempool_net_amount(50000); - w.set_mempool_addresses(vec![addr.clone()]); - } - - manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - - let stored = manager.transactions.get(&txid).unwrap(); - assert_eq!(stored.net_amount, 50000); - assert!(!stored.is_outgoing); - assert!(!stored.is_instant_send); - assert_eq!(stored.addresses.len(), 1); - assert_eq!(stored.addresses[0].to_string(), "yWdXnYxGbouNoo8yMvcbZmZ3Gdp6BpySxL"); - } - - #[tokio::test] - async fn test_handle_tx_outgoing_transaction() { - let (mut manager, _requests, wallet) = create_relevant_manager(); - - let tx = Transaction { - version: 1, - lock_time: 123, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - { - let mut w = wallet.write().await; - w.set_mempool_net_amount(-30000); - } - - manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - - let stored = manager.transactions.get(&txid).unwrap(); - assert_eq!(stored.net_amount, -30000); - assert!(stored.is_outgoing); - assert!(!stored.is_instant_send); - assert!(stored.addresses.is_empty()); - } - - #[test] - fn test_peer_connected_creates_entry() { - let (mut manager, _requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - - assert!(!manager.peers.contains_key(&peer)); - manager.handle_peer_connected(peer); - assert!(manager.peers.contains_key(&peer)); - assert!(manager.peers[&peer].is_none()); - } - - #[test] - fn test_peer_disconnected_redistributes_queue() { - let (mut manager, _requests, _rx) = create_test_manager(); - let peer1 = test_socket_address(1); - let peer2 = test_socket_address(2); - - // Both peers activated with queues - let txid1 = Txid::from_byte_array([1; 32]); - let txid2 = Txid::from_byte_array([2; 32]); - manager.peers.insert(peer1, Some(VecDeque::from([txid1, txid2]))); - manager.peers.insert(peer2, Some(VecDeque::new())); - - manager.handle_peer_disconnected(peer1); - - assert!(!manager.peers.contains_key(&peer1)); - // Txids should have moved to peer2 - let peer2_queue = manager.peers[&peer2].as_ref().unwrap(); - assert!(peer2_queue.contains(&txid1)); - assert!(peer2_queue.contains(&txid2)); - } - - #[test] - fn test_peer_disconnected_no_peers_drops_queue() { - let (mut manager, _requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - - manager.peers.insert(peer, Some(VecDeque::from([Txid::from_byte_array([1; 32])]))); - - manager.handle_peer_disconnected(peer); - - assert!(manager.peers.is_empty()); - } - - #[test] - fn test_prune_pending_requeues_to_activated_peer() { - let (mut manager, _requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - let txid = Txid::from_byte_array([1; 32]); - manager - .pending_requests - .insert(txid, Instant::now() - PENDING_REQUEST_TIMEOUT - Duration::from_secs(1)); - - manager.prune_pending_requests(); - - assert!(!manager.pending_requests.contains_key(&txid)); - assert!(manager.peers[&peer].as_ref().unwrap().contains(&txid)); - } - - #[test] - fn test_prune_pending_drops_when_no_peers() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let txid = Txid::from_byte_array([1; 32]); - manager - .pending_requests - .insert(txid, Instant::now() - PENDING_REQUEST_TIMEOUT - Duration::from_secs(1)); - - manager.prune_pending_requests(); - - assert!(!manager.pending_requests.contains_key(&txid)); - assert!(manager.peers.is_empty()); - } - - #[test] - fn test_remove_confirmed_removes_txids() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let mut txids = Vec::new(); - for i in 0..3u32 { - let tx = Transaction { - version: 1, - lock_time: i, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - txids.push(txid); - manager.transactions.insert( - txid, - UnconfirmedTransaction::new(tx, Amount::from_sat(0), false, false, Vec::new(), 0), - ); - } - assert_eq!(manager.transactions.len(), 3); - // Mark two as recent sends - manager.recent_sends.insert(txids[0], Instant::now()); - manager.recent_sends.insert(txids[1], Instant::now()); - - // Remove 2 of the 3 transactions - manager.remove_confirmed(&txids[..2]); - - assert_eq!(manager.transactions.len(), 1); - assert!(manager.transactions.contains_key(&txids[2])); - assert!(!manager.recent_sends.contains_key(&txids[0])); - assert!(!manager.recent_sends.contains_key(&txids[1])); - - assert_eq!(manager.progress.removed(), 2); - assert_eq!(manager.progress.tracked(), 1); - } - - #[test] - fn test_remove_confirmed_unknown_txids_noop() { - let (mut manager, _requests, _rx) = create_test_manager(); - - let unknown = vec![Txid::from_byte_array([0xaa; 32]), Txid::from_byte_array([0xbb; 32])]; - - manager.remove_confirmed(&unknown); - - assert!(manager.transactions.is_empty()); - assert_eq!(manager.progress.removed(), 0); - } - - #[tokio::test] - async fn test_rebuild_filter_clears_and_reloads() { - let addr = test_address(0xab); - let (mut manager, requests, mut rx) = create_bloom_manager_with_addresses(vec![addr]); - let peer = test_socket_address(1); - - manager.activate_peer(peer, &requests).await.unwrap(); - - // Drain activation messages - while rx.try_recv().is_ok() {} - - manager.rebuild_filter(&requests).await.unwrap(); - - // Verify message sequence: FilterClear, FilterLoad, MemPool - let msg1 = rx.try_recv().unwrap(); - assert!(matches!(msg1, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterClear, _))); - let msg2 = rx.try_recv().unwrap(); - assert!(matches!( - msg2, - NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _) - )); - let msg3 = rx.try_recv().unwrap(); - assert!(matches!(msg3, NetworkRequest::SendMessageToPeer(NetworkMessage::MemPool, _))); - } - - #[tokio::test] - async fn test_rebuild_filter_no_activated_peers_noop() { - let (mut manager, requests, mut rx) = create_bloom_manager(); - // No activation, so no activated peers - assert!(manager.peers.values().all(|v| v.is_none())); - - manager.rebuild_filter(&requests).await.unwrap(); - assert!(rx.try_recv().is_err()); - } - - #[tokio::test] - async fn test_seen_txids_deduplication_window() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.peers.insert(peer, Some(VecDeque::new())); - - let txid = Txid::from_byte_array([1u8; 32]); - let inv = vec![Inventory::Transaction(txid)]; - - // A fresh seen_txids entry should cause handle_inv to skip the txid - manager.seen_txids.insert(txid, Instant::now()); - manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert!(manager.pending_requests.is_empty(), "seen txid should be skipped"); - - // An expired entry should allow the txid to be accepted again - manager.seen_txids.insert(txid, Instant::now() - SEEN_TXID_EXPIRY - Duration::from_secs(1)); - manager.handle_inv(&inv, peer, &requests).await.unwrap(); - assert!( - manager.pending_requests.contains_key(&txid), - "expired seen txid should be accepted" - ); - } - - #[tokio::test] - async fn test_rebroadcast_sends_old_recent_sends() { - let (mut manager, requests, mut rx) = create_test_manager(); - - let tx = Transaction { - version: 10, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - let t0 = Instant::now(); - let later = t0 + REBROADCAST_INTERVAL + Duration::from_secs(1); - - manager.transactions.insert( - txid, - UnconfirmedTransaction::new(tx, Amount::from_sat(0), false, true, Vec::new(), -100_000), - ); - manager.recent_sends.insert(txid, t0); - - manager.rebroadcast_if_due_at(&requests, later).await; - - // Should have sent a BroadcastMessage for the transaction - let msg = rx.try_recv().expect("expected a rebroadcast message"); - assert!( - matches!(msg, NetworkRequest::BroadcastMessage(NetworkMessage::Tx(_))), - "expected BroadcastMessage(Tx), got {:?}", - msg - ); - - // Timestamp should be reset to `later`, so a second call at the same instant - // must not rebroadcast. - manager.rebroadcast_if_due_at(&requests, later).await; - assert!(rx.try_recv().is_err(), "should not rebroadcast immediately after reset"); - } - - #[tokio::test] - async fn test_rebroadcast_skips_recent_transactions() { - let (mut manager, requests, mut rx) = create_test_manager(); - - let tx = Transaction { - version: 11, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - - // Add a transaction that was just sent (within the rebroadcast interval) - manager.transactions.insert( - txid, - UnconfirmedTransaction::new(tx, Amount::from_sat(0), false, true, Vec::new(), -50_000), - ); - manager.recent_sends.insert(txid, Instant::now()); - - manager.rebroadcast_if_due(&requests).await; - - assert!(rx.try_recv().is_err(), "recently sent transactions should not be rebroadcast"); - } - - #[test] - fn test_peer_disconnect_keeps_other_peers_intact() { - let (mut manager, _requests, _rx) = create_test_manager(); - let peer1 = test_socket_address(1); - let peer2 = test_socket_address(2); - - // Both activated - manager.peers.insert(peer1, Some(VecDeque::new())); - manager.peers.insert(peer2, Some(VecDeque::from([Txid::from_byte_array([1; 32])]))); - - manager.handle_peer_disconnected(peer1); - - assert!(!manager.peers.contains_key(&peer1)); - // peer2 should still be present and activated - assert!(manager.peers.contains_key(&peer2)); - assert!(manager.peers[&peer2].is_some()); - } -} diff --git a/dash-spv/src/sync/mempool/sync_manager.rs b/dash-spv/src/sync/mempool/sync_manager.rs index f818ed42c..47ec1a406 100644 --- a/dash-spv/src/sync/mempool/sync_manager.rs +++ b/dash-spv/src/sync/mempool/sync_manager.rs @@ -1,12 +1,14 @@ use super::manager::MEMPOOL_TX_EXPIRY; use crate::error::SyncResult; -use crate::network::{Message, MessageType, NetworkEvent, RequestSender}; +use crate::network2::PeerNetworkManager; use crate::sync::{ ManagerIdentifier, MempoolManager, SyncEvent, SyncManager, SyncManagerProgress, SyncState, }; use async_trait::async_trait; use dashcore::network::message::NetworkMessage; use key_wallet_manager::WalletInterface; +use std::net::SocketAddr; +use std::sync::Arc; #[async_trait] impl SyncManager for MempoolManager { @@ -22,13 +24,17 @@ impl SyncManager for MempoolManager { self.progress.set_state(state); } - fn wanted_message_types(&self) -> &'static [MessageType] { + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { + use crate::network2::MessageType; &[MessageType::Inv, MessageType::Tx] } - async fn start_sync(&mut self, requests: &RequestSender) -> SyncResult> { + async fn start_sync( + &mut self, + network: &Arc, + ) -> SyncResult> { // After a full disconnect, re-activate mempool on all connected peers - self.activate_all_peers(requests).await?; + self.activate_all_peers(network).await?; let has_activated = self.peers.values().any(|v| v.is_some()); if has_activated { self.set_state(SyncState::Synced); @@ -45,12 +51,13 @@ impl SyncManager for MempoolManager { async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult> { - match msg.inner() { - NetworkMessage::Inv(inv) => self.handle_inv(inv, msg.peer_address(), requests).await, - NetworkMessage::Tx(tx) => self.handle_tx(tx.clone(), msg.peer_address()).await, + match &msg { + NetworkMessage::Inv(inv) => self.handle_inv(inv, peer, network).await, + NetworkMessage::Tx(tx) => self.handle_tx((**tx).clone(), peer).await, _ => Ok(vec![]), } } @@ -58,7 +65,7 @@ impl SyncManager for MempoolManager { async fn handle_sync_event( &mut self, event: &SyncEvent, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { match event { // Activate as soon as filter sync completes — the wallet's address @@ -67,7 +74,7 @@ impl SyncManager for MempoolManager { .. } => { if self.state() != SyncState::Synced { - self.activate_all_peers(requests).await?; + self.activate_all_peers(network).await?; let has_activated = self.peers.values().any(|v| v.is_some()); if has_activated { self.set_state(SyncState::Synced); @@ -103,7 +110,7 @@ impl SyncManager for MempoolManager { } } - async fn tick(&mut self, requests: &RequestSender) -> SyncResult> { + async fn tick(&mut self, network: &Arc) -> SyncResult> { if self.state() != SyncState::Synced { return Ok(vec![]); } @@ -115,10 +122,10 @@ impl SyncManager for MempoolManager { self.prune_pending_requests(); // Send queued getdata requests now that slots may have freed up - self.send_queued(requests).await?; + self.send_queued(network).await?; // Rebroadcast unconfirmed self-sent transactions on a randomized interval - self.rebroadcast_if_due(requests).await; + self.rebroadcast_if_due(network).await; // Rebuild bloom filter if the wallet's monitored set has changed. // @@ -133,699 +140,14 @@ impl SyncManager for MempoolManager { let current_revision = self.wallet.read().await.monitor_revision(); if current_revision != self.last_monitor_revision { tracing::info!("Wallet monitor revision changed, rebuilding bloom filter"); - self.rebuild_filter(requests).await?; + self.rebuild_filter(network).await?; self.last_monitor_revision = current_revision; } Ok(vec![]) } - async fn handle_network_event( - &mut self, - event: &NetworkEvent, - requests: &RequestSender, - ) -> SyncResult> { - match event { - NetworkEvent::PeerConnected { - address, - } => { - self.handle_peer_connected(*address); - // If synced, activate the new peer immediately - if self.state() == SyncState::Synced - && self.peers.get(address).is_some_and(|v| v.is_none()) - { - tracing::info!("Activating mempool on newly connected peer {}", address); - self.activate_peer(*address, requests).await?; - } - } - NetworkEvent::PeerDisconnected { - address, - } => { - self.handle_peer_disconnected(*address); - } - NetworkEvent::PeersUpdated { - connected_count, - best_height, - .. - } => { - if let Some(best_height) = best_height { - self.update_target_height(*best_height); - } - if *connected_count == 0 { - self.stop_sync(); - } else if self.state() == SyncState::WaitingForConnections { - return self.start_sync(requests).await; - } - } - } - Ok(vec![]) - } - fn progress(&self) -> SyncManagerProgress { SyncManagerProgress::Mempool(self.progress.clone()) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::client::config::MempoolStrategy; - use crate::network::NetworkRequest; - use crate::test_utils::test_socket_address; - use dashcore::hashes::Hash; - use key_wallet_manager::test_utils::MockWallet; - use std::collections::{BTreeMap, BTreeSet}; - use std::sync::Arc; - use tokio::sync::{mpsc, RwLock}; - - fn create_test_manager( - ) -> (MempoolManager, RequestSender, mpsc::UnboundedReceiver) { - let wallet = Arc::new(RwLock::new(MockWallet::new())); - let (tx, rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let manager = MempoolManager::new(wallet, MempoolStrategy::FetchAll, 1000, 0); - - (manager, requests, rx) - } - - #[test] - fn test_sync_manager_trait_basics() { - let (mut manager, _, _rx) = create_test_manager(); - - assert_eq!(manager.identifier(), ManagerIdentifier::Mempool); - assert_eq!(manager.state(), SyncState::WaitForEvents); - - let types = manager.wanted_message_types(); - assert!(types.contains(&MessageType::Inv)); - assert!(types.contains(&MessageType::Tx)); - assert_eq!(types.len(), 2); - - manager.set_state(SyncState::Synced); - assert_eq!(manager.state(), SyncState::Synced); - - assert!(matches!(manager.progress(), SyncManagerProgress::Mempool(_))); - } - - #[tokio::test] - async fn test_filters_sync_complete_activates() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = crate::test_utils::test_socket_address(1); - manager.handle_peer_connected(peer); - - let event = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - - let events = manager.handle_sync_event(&event, &requests).await.unwrap(); - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::Synced); - assert!(matches!(manager.peers.get(&peer), Some(Some(_)))); - } - - #[tokio::test] - async fn test_filters_sync_complete_subsequent_is_noop() { - let (mut manager, requests, _rx) = create_test_manager(); - manager.handle_peer_connected(crate::test_utils::test_socket_address(1)); - - // Activate first - let event0 = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&event0, &requests).await.unwrap(); - - // Subsequent filter sync completions should not change state - let event1 = SyncEvent::FiltersSyncComplete { - tip_height: 1001, - }; - let events = manager.handle_sync_event(&event1, &requests).await.unwrap(); - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::Synced); - } - - #[tokio::test] - async fn test_reactivation_after_disconnect() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - // Initial activation - let event = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - let events = manager.handle_sync_event(&event, &requests).await.unwrap(); - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::Synced); - - // Simulate disconnect by resetting state - manager.set_state(SyncState::WaitForEvents); - - // Re-sync should re-activate - let event = SyncEvent::FiltersSyncComplete { - tip_height: 1001, - }; - let events = manager.handle_sync_event(&event, &requests).await.unwrap(); - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::Synced); - } - - #[tokio::test] - async fn test_peer_connect_activates_when_synced() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer1 = test_socket_address(1); - manager.handle_peer_connected(peer1); - - // Activate via SyncComplete - let event = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&event, &requests).await.unwrap(); - assert!(matches!(manager.peers.get(&peer1), Some(Some(_)))); - - // New peer connects while synced => should activate immediately - let peer2 = test_socket_address(2); - let connect = NetworkEvent::PeerConnected { - address: peer2, - }; - let events = manager.handle_network_event(&connect, &requests).await.unwrap(); - assert!(events.is_empty()); - assert!(matches!(manager.peers.get(&peer2), Some(Some(_)))); - } - - #[tokio::test] - async fn test_network_event_peer_connect_disconnect() { - let (mut manager, requests, _rx) = create_test_manager(); - - let peer1 = test_socket_address(1); - let peer2 = test_socket_address(2); - - // Connecting peers should return empty events (not synced yet) - let connect1 = NetworkEvent::PeerConnected { - address: peer1, - }; - let events = manager.handle_network_event(&connect1, &requests).await.unwrap(); - assert!(events.is_empty()); - assert!(manager.peers.contains_key(&peer1)); - - let connect2 = NetworkEvent::PeerConnected { - address: peer2, - }; - let events = manager.handle_network_event(&connect2, &requests).await.unwrap(); - assert!(events.is_empty()); - assert_eq!(manager.peers.len(), 2); - - let disconnect1 = NetworkEvent::PeerDisconnected { - address: peer1, - }; - let events = manager.handle_network_event(&disconnect1, &requests).await.unwrap(); - assert!(events.is_empty()); - - // Still have peer2 available - assert!(manager.peers.contains_key(&peer2)); - assert_eq!(manager.peers.len(), 1); - - // Disconnecting an already-disconnected peer should not error - let events = manager.handle_network_event(&disconnect1, &requests).await.unwrap(); - assert!(events.is_empty()); - } - - #[tokio::test] - async fn test_block_processed_removes_confirmed_txids() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - // Activate - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - - // Add transactions to mempool - let mut txids = Vec::new(); - for i in 0..2u32 { - let tx = dashcore::Transaction { - version: 1, - lock_time: i, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - txids.push(txid); - manager.transactions.insert( - txid, - crate::types::UnconfirmedTransaction::new( - tx, - dashcore::Amount::from_sat(0), - false, - false, - Vec::new(), - 0, - ), - ); - } - - let event = SyncEvent::BlockProcessed { - block_hash: dashcore::BlockHash::all_zeros(), - height: 1001, - wallets: BTreeSet::new(), - new_scripts: BTreeMap::new(), - confirmed_txids: txids.clone(), - }; - let events = manager.handle_sync_event(&event, &requests).await.unwrap(); - assert!(events.is_empty()); - - assert!(manager.transactions.is_empty()); - } - - #[tokio::test] - async fn test_instant_lock_received_marks_transaction() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - // Activate - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - - // Add a transaction to mempool - let tx = dashcore::Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - let txid = tx.txid(); - manager.transactions.insert( - txid, - crate::types::UnconfirmedTransaction::new( - tx, - dashcore::Amount::from_sat(0), - false, - false, - Vec::new(), - 0, - ), - ); - - // Fire InstantLockReceived with a lock whose txid matches - let mut is_lock = dashcore::InstantLock::dummy(0..1); - is_lock.txid = txid; - - let event = SyncEvent::InstantLockReceived { - instant_lock: is_lock, - validated: true, - }; - let events = manager.handle_sync_event(&event, &requests).await.unwrap(); - assert!(events.is_empty()); - - assert!(manager.transactions.get(&txid).unwrap().is_instant_send); - } - - #[tokio::test] - async fn test_peer_disconnect_removes_from_peers() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - // Activate - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - - // Disconnect the only peer - let disconnect = NetworkEvent::PeerDisconnected { - address: peer, - }; - let events = manager.handle_network_event(&disconnect, &requests).await.unwrap(); - assert!(events.is_empty()); - assert!(manager.peers.is_empty()); - } - - #[tokio::test] - async fn test_sync_complete_no_peers_stays_inactive() { - let (mut manager, requests, _rx) = create_test_manager(); - - let event = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - let events = manager.handle_sync_event(&event, &requests).await.unwrap(); - - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::WaitForEvents); - assert!(manager.peers.is_empty()); - } - - #[tokio::test] - async fn test_start_sync_no_peers_stays_waiting() { - let (mut manager, requests, _rx) = create_test_manager(); - - // Simulate full disconnect setting state to WaitingForConnections - manager.set_state(SyncState::WaitingForConnections); - - // start_sync with no peers should stay in WaitingForConnections - let events = manager.start_sync(&requests).await.unwrap(); - assert!(events.is_empty()); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - } - - #[tokio::test] - async fn test_disconnect_recovery_reactivates_on_reconnect() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - // Activate via SyncComplete - let event = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&event, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Synced); - - // Disconnect peer - let disconnect = NetworkEvent::PeerDisconnected { - address: peer, - }; - manager.handle_network_event(&disconnect, &requests).await.unwrap(); - - // PeersUpdated with 0 triggers stop_sync - let update = NetworkEvent::PeersUpdated { - connected_count: 0, - addresses: vec![], - best_height: None, - }; - manager.handle_network_event(&update, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - - // PeersUpdated with 1 but no peers tracked yet: stays WaitingForConnections - let update = NetworkEvent::PeersUpdated { - connected_count: 1, - addresses: vec![peer], - best_height: Some(1000), - }; - manager.handle_network_event(&update, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::WaitingForConnections); - - // Peer reconnects and PeersUpdated fires again - manager.handle_peer_connected(peer); - let update = NetworkEvent::PeersUpdated { - connected_count: 1, - addresses: vec![peer], - best_height: Some(1000), - }; - manager.handle_network_event(&update, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Synced); - assert!(matches!(manager.peers.get(&peer), Some(Some(_)))); - } - - #[tokio::test] - async fn test_block_processed_confirmed_txids_does_not_eagerly_rebuild() { - let mut mock = MockWallet::new(); - let script = dashcore::ScriptBuf::from_bytes(vec![ - 0x76, 0xa9, 0x14, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, - 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0x88, 0xac, - ]); - let addr = dashcore::Address::from_script(&script, dashcore::Network::Testnet).unwrap(); - mock.set_addresses(vec![addr]); - let wallet = Arc::new(RwLock::new(mock)); - let (tx, mut rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new(wallet, MempoolStrategy::BloomFilter, 1000, 0); - - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - // Activate - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - - // Drain activation messages - while rx.try_recv().is_ok() {} - - // BlockProcessed does not eagerly rebuild — the tick handles it via - // the revision check. Verify no FilterLoad is sent from the event handler. - let event = SyncEvent::BlockProcessed { - block_hash: dashcore::BlockHash::all_zeros(), - height: 1001, - wallets: BTreeSet::new(), - new_scripts: BTreeMap::new(), - confirmed_txids: vec![dashcore::Txid::all_zeros()], - }; - manager.handle_sync_event(&event, &requests).await.unwrap(); - - let has_filter_load = std::iter::from_fn(|| rx.try_recv().ok()).any(|req| { - matches!(req, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _)) - }); - assert!(!has_filter_load, "BlockProcessed should not eagerly rebuild filter"); - } - - #[tokio::test] - async fn test_block_processed_no_changes_no_rebuild_flag() { - let (mut manager, requests, _rx) = create_test_manager(); - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - - // BlockProcessed with no confirmed txids and no new addresses - let event = SyncEvent::BlockProcessed { - block_hash: dashcore::BlockHash::all_zeros(), - height: 1001, - wallets: BTreeSet::new(), - new_scripts: BTreeMap::new(), - confirmed_txids: vec![], - }; - manager.handle_sync_event(&event, &requests).await.unwrap(); - } - - #[tokio::test] - async fn test_tick_rebuilds_filter_when_monitor_revision_changes() { - let addr = { - let script = dashcore::ScriptBuf::from_bytes(vec![ - 0x76, 0xa9, 0x14, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, - 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0x88, 0xac, - ]); - dashcore::Address::from_script(&script, dashcore::Network::Testnet).unwrap() - }; - - let mut mock = MockWallet::new(); - mock.set_addresses(vec![addr.clone()]); - let initial_revision = mock.monitor_revision(); - let wallet = Arc::new(RwLock::new(mock)); - let (tx, mut rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new( - wallet.clone(), - MempoolStrategy::BloomFilter, - 1000, - initial_revision, - ); - - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - // Activate — this snapshots the monitor revision - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - assert_eq!(manager.state(), SyncState::Synced); - - // Drain activation messages - while rx.try_recv().is_ok() {} - - // tick with unchanged revision should not rebuild - manager.tick(&requests).await.unwrap(); - assert!(rx.try_recv().is_err(), "no messages expected when revision unchanged"); - - // Simulate wallet adding new addresses (bumps revision) - { - let mut w = wallet.write().await; - let addr2 = dashcore::Address::from_script( - &dashcore::ScriptBuf::from_bytes(vec![ - 0x76, 0xa9, 0x14, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, - 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0xcd, 0x88, 0xac, - ]), - dashcore::Network::Testnet, - ) - .unwrap(); - w.set_addresses(vec![addr, addr2]); - } - - // tick should detect stale filter and rebuild - manager.tick(&requests).await.unwrap(); - - let mut found_filter_load = false; - while let Ok(msg) = rx.try_recv() { - if matches!(msg, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _)) { - found_filter_load = true; - } - } - assert!(found_filter_load, "expected FilterLoad after monitor revision change"); - - // Subsequent tick should not rebuild again (revision was snapshotted) - manager.tick(&requests).await.unwrap(); - assert!(rx.try_recv().is_err(), "no messages expected after revision re-snapshot"); - } - - #[tokio::test] - async fn test_tick_skips_rebuild_for_fetch_all_strategy() { - let wallet = Arc::new(RwLock::new(MockWallet::new())); - let (tx, mut rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new(wallet.clone(), MempoolStrategy::FetchAll, 1000, 0); - - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - while rx.try_recv().is_ok() {} - - // Bump revision - { - let mut w = wallet.write().await; - w.set_addresses(vec![dashcore::Address::dummy(dashcore::Network::Testnet, 0)]); - } - - // tick should not send any filter messages for FetchAll - manager.tick(&requests).await.unwrap(); - let mut found_filter = false; - while let Ok(msg) = rx.try_recv() { - if matches!( - msg, - NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _) - | NetworkRequest::SendMessageToPeer(NetworkMessage::FilterClear, _) - ) { - found_filter = true; - } - } - assert!(!found_filter, "FetchAll should not send filter messages on revision change"); - } - - #[tokio::test] - async fn test_tick_rebuilds_filter_when_outpoints_change() { - let addr = { - let script = dashcore::ScriptBuf::from_bytes(vec![ - 0x76, 0xa9, 0x14, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, - 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0x88, 0xac, - ]); - dashcore::Address::from_script(&script, dashcore::Network::Testnet).unwrap() - }; - - let mut mock = MockWallet::new(); - mock.set_addresses(vec![addr]); - let initial_revision = mock.monitor_revision(); - let wallet = Arc::new(RwLock::new(mock)); - let (tx, mut rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx); - - let mut manager = MempoolManager::new( - wallet.clone(), - MempoolStrategy::BloomFilter, - 1000, - initial_revision, - ); - - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - while rx.try_recv().is_ok() {} - - // Simulate UTXO set change (new outpoint added) - { - let mut w = wallet.write().await; - w.set_outpoints(vec![dashcore::OutPoint { - txid: dashcore::Txid::from_byte_array([0xee; 32]), - vout: 0, - }]); - } - - // tick should detect the revision change and rebuild - manager.tick(&requests).await.unwrap(); - - let found_filter_load = std::iter::from_fn(|| rx.try_recv().ok()).any(|msg| { - matches!(msg, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _)) - }); - assert!(found_filter_load, "expected FilterLoad after outpoint change"); - } - - #[tokio::test] - async fn test_handle_tx_does_not_eagerly_rebuild_filter() { - let mut mock = MockWallet::new(); - mock.set_mempool_relevant(true); - let script = dashcore::ScriptBuf::from_bytes(vec![ - 0x76, 0xa9, 0x14, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, - 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0xab, 0x88, 0xac, - ]); - let addr = dashcore::Address::from_script(&script, dashcore::Network::Testnet).unwrap(); - mock.set_addresses(vec![addr]); - let initial_revision = mock.monitor_revision(); - let wallet = Arc::new(RwLock::new(mock)); - let (tx_chan, mut rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(tx_chan); - - let mut manager = MempoolManager::new( - wallet.clone(), - MempoolStrategy::BloomFilter, - 1000, - initial_revision, - ); - - let peer = test_socket_address(1); - manager.handle_peer_connected(peer); - - let sync = SyncEvent::FiltersSyncComplete { - tip_height: 1000, - }; - manager.handle_sync_event(&sync, &requests).await.unwrap(); - while rx.try_recv().is_ok() {} - - // handle_tx with a relevant transaction should NOT eagerly rebuild - let tx = dashcore::Transaction { - version: 1, - lock_time: 0, - input: vec![], - output: vec![], - special_transaction_payload: None, - }; - manager.handle_tx(tx, test_socket_address(1)).await.unwrap(); - - let has_filter_load = std::iter::from_fn(|| rx.try_recv().ok()).any(|msg| { - matches!(msg, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _)) - }); - assert!(!has_filter_load, "handle_tx should not eagerly rebuild filter"); - - // But the next tick should catch it if the wallet revision changed - // (MockWallet bumps revision when set_mempool_relevant triggers processing) - { - let mut w = wallet.write().await; - w.set_addresses(vec![dashcore::Address::dummy(dashcore::Network::Testnet, 0)]); - } - manager.tick(&requests).await.unwrap(); - - let found_filter_load = std::iter::from_fn(|| rx.try_recv().ok()).any(|msg| { - matches!(msg, NetworkRequest::SendMessageToPeer(NetworkMessage::FilterLoad(_), _)) - }); - assert!(found_filter_load, "tick should rebuild after revision change"); - } -} diff --git a/dash-spv/src/sync/mod.rs b/dash-spv/src/sync/mod.rs index 8a6925ef7..9d97ebe82 100644 --- a/dash-spv/src/sync/mod.rs +++ b/dash-spv/src/sync/mod.rs @@ -3,7 +3,7 @@ mod block_headers; mod blocks; mod chainlock; -pub(super) mod download_coordinator; +mod download_coordinator; mod events; mod filter_headers; mod filters; diff --git a/dash-spv/src/sync/sync_coordinator.rs b/dash-spv/src/sync/sync_coordinator.rs index 7f9bc245d..7b83c8a95 100644 --- a/dash-spv/src/sync/sync_coordinator.rs +++ b/dash-spv/src/sync/sync_coordinator.rs @@ -13,7 +13,7 @@ use tokio_stream::wrappers::WatchStream; use tokio_util::sync::CancellationToken; use crate::error::SyncResult; -use crate::network::NetworkManager; +use crate::network2::PeerNetworkManager; use crate::storage::{ BlockHeaderStorage, BlockStorage, FilterHeaderStorage, FilterStorage, MetadataStorage, }; @@ -25,6 +25,7 @@ use crate::sync::{ }; use crate::SyncError; use key_wallet_manager::WalletInterface; +use std::sync::Arc; const TASK_JOIN_TIMEOUT: Duration = Duration::from_secs(5); const DEFAULT_SYNC_EVENT_CAPACITY: usize = 10000; @@ -34,23 +35,18 @@ macro_rules! spawn_manager { ($self:expr, $manager:expr, $network:expr) => { if let Some(manager) = $manager { let identifier = manager.identifier(); - let wanted_message_types = manager.wanted_message_types(); - let requests = $network.request_sender(); - let message_receiver = $network.message_receiver(wanted_message_types).await; - let network_event_rx = $network.subscribe_network_events(); + let commands = manager.subscribed_commands(); + let inbound = $network.subscribe(commands).await; + let network_event_rx = $network.events(); let (progress_sender, progress_receiver) = watch::channel(manager.progress()); - tracing::info!( - "Spawning {} task, receiving message types: {:?}", - identifier, - wanted_message_types - ); + tracing::info!("Spawning {} task, subscribing to commands: {:?}", identifier, commands); let context = SyncManagerTaskContext { - message_receiver, + inbound, sync_event_sender: $self.sync_event_sender.clone(), network_event_receiver: network_event_rx, - requests, + network: $network.clone(), shutdown: $self.shutdown.clone(), progress_sender, }; @@ -194,10 +190,7 @@ where /// - An event bus subscription for inter-manager events /// - A request sender for outgoing network messages /// - A shutdown token for graceful termination - pub async fn start(&mut self, network: &mut N) -> SyncResult<()> - where - N: NetworkManager, - { + pub async fn start(&mut self, network: &Arc) -> SyncResult<()> { if !self.tasks.is_empty() { return Err(SyncError::InvalidState("SyncCoordinator already started".to_string())); } @@ -477,10 +470,13 @@ mod tests { filter_headers_progress.add_processed(500); progress.update_filter_headers(filter_headers_progress); - // Create filters progress at 25% + // Create filters progress at 25%. Filter progress is how far the filters + // themselves are DOWNLOADED (stored), not how far the scan has committed — + // the commit frontier trails the download by a long way. let mut filters_progress = FiltersProgress::default(); filters_progress.set_state(SyncState::Syncing); - filters_progress.update_committed_height(250); + filters_progress.update_stored_height(250); + filters_progress.update_committed_height(100); filters_progress.update_target_height(1000); filters_progress.add_downloaded(250); progress.update_filters(filters_progress); diff --git a/dash-spv/src/sync/sync_manager.rs b/dash-spv/src/sync/sync_manager.rs index 997e9f505..340ce1474 100644 --- a/dash-spv/src/sync/sync_manager.rs +++ b/dash-spv/src/sync/sync_manager.rs @@ -1,11 +1,14 @@ use crate::error::SyncResult; -use crate::network::{Message, MessageType, NetworkEvent, RequestSender}; +use crate::network2::{NetworkEvent, PeerNetworkManager, RequestKey}; use crate::sync::{ BlockHeadersProgress, BlocksProgress, ChainLockProgress, FilterHeadersProgress, FiltersProgress, InstantSendProgress, ManagerIdentifier, MasternodesProgress, MempoolProgress, SyncEvent, SyncState, }; use async_trait::async_trait; +use dashcore::network::message::NetworkMessage; +use std::net::SocketAddr; +use std::sync::Arc; use crate::SyncError; @@ -48,11 +51,13 @@ impl SyncManagerProgress { } } +pub type Inbound = (SocketAddr, Arc); + pub struct SyncManagerTaskContext { - pub(super) message_receiver: UnboundedReceiver, + pub(super) inbound: UnboundedReceiver, pub(super) sync_event_sender: broadcast::Sender, pub(super) network_event_receiver: broadcast::Receiver, - pub(super) requests: RequestSender, + pub(super) network: Arc, pub(super) shutdown: CancellationToken, pub(super) progress_sender: watch::Sender, } @@ -68,6 +73,19 @@ impl SyncManagerTaskContext { } } +// Display for the network2 event so the sync loop / broadcast monitor can log +// it (they require `Display`). Kept here to avoid modifying the network2 module. +impl std::fmt::Display for NetworkEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NetworkEvent::PeersUpdated => write!(f, "PeersUpdated"), + NetworkEvent::PeerConnected(addr) => write!(f, "PeerConnected({addr})"), + NetworkEvent::PeerDisconnected(addr) => write!(f, "PeerDisconnected({addr})"), + NetworkEvent::RequestOnFlight(_) => write!(f, "RequestOnFlight"), + } + } +} + /// Guard that verifies a manager has not already been started. pub(super) fn ensure_not_started( state: SyncState, @@ -98,14 +116,17 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { /// /// The network manager uses this to route only relevant messages to each /// manager's task via topic-based filtering. - fn wanted_message_types(&self) -> &'static [MessageType]; + fn subscribed_commands(&self) -> &'static [crate::network2::MessageType]; /// Start the sync process. /// /// Called after initialization to trigger the initial sync requests. /// For example, BlockHeadersManager sends its first getheaders request here. /// The default implementation is for reactive managers that just wait for events. - async fn start_sync(&mut self, _requests: &RequestSender) -> SyncResult> { + async fn start_sync( + &mut self, + _network: &Arc, + ) -> SyncResult> { ensure_not_started(self.state(), self.identifier())?; self.set_state(SyncState::WaitForEvents); Ok(vec![SyncEvent::SyncStart { @@ -138,8 +159,9 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { /// Returns events to emit to other managers. async fn handle_message( &mut self, - msg: Message, - requests: &RequestSender, + peer: SocketAddr, + msg: NetworkMessage, + network: &Arc, ) -> SyncResult>; /// Handle a sync event from another manager. @@ -150,7 +172,7 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { async fn handle_sync_event( &mut self, event: &SyncEvent, - requests: &RequestSender, + network: &Arc, ) -> SyncResult>; /// Periodic tick for timeouts, retries, and proactive work. @@ -160,7 +182,7 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { /// - Timeout detection and retry logic /// - Proactive request sending /// - State cleanup - async fn tick(&mut self, requests: &RequestSender) -> SyncResult>; + async fn tick(&mut self, network: &Arc) -> SyncResult>; /// Handle a network event (peer connection changes). /// @@ -169,33 +191,34 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { async fn handle_network_event( &mut self, event: &NetworkEvent, - requests: &RequestSender, + network: &Arc, ) -> SyncResult> { - // Default: transition from WaitingForConnections to Syncing when peers connect - if let NetworkEvent::PeersUpdated { - connected_count, - best_height, - .. - } = event - { - if let Some(best_height) = best_height { - self.update_target_height(*best_height); - } - if *connected_count == 0 { - tracing::info!("{} - no peers available, stopping sync", self.identifier()); - self.stop_sync(); - } else if *connected_count > 0 && self.state() == SyncState::WaitingForConnections { - tracing::info!( - "{} - peers available ({}), starting sync", - self.identifier(), - connected_count - ); - return self.start_sync(requests).await; + // Peers are connected before the manager starts; PeersUpdated is our + // cue to kick off the initial requests. Individual peer disconnects are + // recovered by per-request timeout+retry, so they don't stop sync. + if let NetworkEvent::PeersUpdated = event { + // Seed every manager's target from the peers' advertised tip so the + // height shows up right away (matches the pre-network2 `best_height`). + self.update_target_height(network.tip()); + if self.state() == SyncState::WaitingForConnections { + tracing::info!("{} - peers available, starting sync", self.identifier()); + return self.start_sync(network).await; } } + // A request just went on the wire: let the owning pipeline start its + // response timeout from now (see `mark_on_flight`). + if let NetworkEvent::RequestOnFlight(key) = event { + self.mark_on_flight(key); + } Ok(vec![]) } + /// Start the response timeout for a request the network manager just put on + /// the wire. Default no-op; each pipeline-owning manager overrides it to + /// forward the matching `RequestKey` variant to its coordinator's + /// `mark_on_flight`. Keeps retry counting from actual send, not from queue. + fn mark_on_flight(&mut self, _key: &RequestKey) {} + /// Retrieves the current progress of the Manager. fn progress(&self) -> SyncManagerProgress; @@ -225,6 +248,21 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { // Tick interval for periodic housekeeping let mut tick_interval = interval(Duration::from_millis(100)); + // Peers are already connected when this task starts: network2 connects + // at construction, and its one-shot PeersUpdated may fire before we + // subscribe. So replay it now (seeds the target height and kicks off + // sync) instead of waiting for the event. + { + let progress_before = self.progress(); + match self.handle_network_event(&NetworkEvent::PeersUpdated, &context.network).await { + Ok(events) => { + context.emit_sync_events(events); + self.try_emit_progress(progress_before, &context.progress_sender); + } + Err(e) => tracing::debug!("{} initial start_sync skipped: {}", identifier, e), + } + } + tracing::info!("{} task entering main loop", identifier); loop { @@ -234,10 +272,16 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { break; } // Process incoming network messages - Some(message) = context.message_receiver.recv() => { + Some((peer, message)) = context.inbound.recv() => { tracing::trace!("{} received message: {}", identifier, message.cmd()); + // The pump gives its last subscriber the sole reference, so for the + // message types only one manager watches — block, cfilter, cfheaders, + // headers — this takes the payload without copying it. A genuinely + // shared message (`inv` goes to several managers) falls back to a clone. + let message = + Arc::try_unwrap(message).unwrap_or_else(|shared| (*shared).clone()); let progress_before = self.progress(); - match self.handle_message(message, &context.requests).await { + match self.handle_message(peer, message, &context.network).await { Ok(events) => { if !events.is_empty() { for event in &events { @@ -263,7 +307,7 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { Ok(event) => { tracing::trace!("{} received event: {}", identifier, event); let progress_before = self.progress(); - match self.handle_sync_event(&event, &context.requests).await { + match self.handle_sync_event(&event, &context.network).await { Ok(events) => { if !events.is_empty() { for e in &events { @@ -278,6 +322,13 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { } } } + Err(broadcast::error::RecvError::Lagged(n)) => { + // Sync-event bus overflowed for this manager; skipped `n` + // events. Keep running rather than killing the task — a + // dropped event is recoverable via tick()/reconciliation, + // a dead task is not. + tracing::warn!("{} lagged sync events, skipped {}", identifier, n); + } Err(error) => { tracing::error!("{} sync event error: {}", identifier, error); break; @@ -290,7 +341,7 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { Ok(event) => { tracing::debug!("{} received network event: {}", identifier, event); let progress_before = self.progress(); - match self.handle_network_event(&event, &context.requests).await { + match self.handle_network_event(&event, &context.network).await { Ok(events) => { if !events.is_empty() { for e in &events { @@ -305,6 +356,13 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { } } } + Err(broadcast::error::RecvError::Lagged(n)) => { + // Network-event bus overflowed (RequestOnFlight is + // high-volume). Dropping one is harmless — the request + // falls back to its queue-grace timeout — so keep + // running instead of killing the manager. + tracing::warn!("{} lagged network events, skipped {}", identifier, n); + } Err(error) => { tracing::error!("{} network event error: {}", identifier, error); break; @@ -314,7 +372,7 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { // Periodic tick for timeouts and housekeeping _ = tick_interval.tick() => { let progress_before = self.progress(); - match self.tick(&context.requests).await { + match self.tick(&context.network).await { Ok(events) => { if !events.is_empty() { context.emit_sync_events(events); @@ -333,132 +391,3 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { Ok(identifier) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::network::NetworkRequest; - use crate::sync::BlockHeadersProgress; - use crate::sync::SyncState; - use async_trait::async_trait; - use std::sync::atomic::{AtomicU32, Ordering}; - use std::sync::Arc; - use tokio::sync::{broadcast, mpsc}; - - /// Mock manager for testing the task runner. - struct MockManager { - identifier: ManagerIdentifier, - state: SyncState, - message_count: Arc, - event_count: Arc, - tick_count: Arc, - } - - impl std::fmt::Debug for MockManager { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MockManager").field("identifier", &self.identifier).finish() - } - } - - #[async_trait] - impl SyncManager for MockManager { - fn identifier(&self) -> ManagerIdentifier { - self.identifier - } - - fn state(&self) -> SyncState { - self.state - } - - fn set_state(&mut self, state: SyncState) { - self.state = state; - } - - fn wanted_message_types(&self) -> &'static [MessageType] { - &[] - } - - fn on_disconnect(&mut self) {} - - async fn handle_message( - &mut self, - _msg: Message, - _requests: &RequestSender, - ) -> SyncResult> { - self.message_count.fetch_add(1, Ordering::Relaxed); - Ok(vec![]) - } - - async fn handle_sync_event( - &mut self, - _event: &SyncEvent, - _requests: &RequestSender, - ) -> SyncResult> { - self.event_count.fetch_add(1, Ordering::Relaxed); - Ok(vec![]) - } - - async fn tick(&mut self, _requests: &RequestSender) -> SyncResult> { - self.tick_count.fetch_add(1, Ordering::Relaxed); - Ok(vec![]) - } - - fn progress(&self) -> SyncManagerProgress { - let mut progress = BlockHeadersProgress::default(); - progress.set_state(self.state); - SyncManagerProgress::BlockHeaders(progress) - } - } - - #[tokio::test] - async fn test_manager_task_shutdown() { - let message_count = Arc::new(AtomicU32::new(0)); - let event_count = Arc::new(AtomicU32::new(0)); - let tick_count = Arc::new(AtomicU32::new(0)); - - let manager = MockManager { - identifier: ManagerIdentifier::BlockHeader, - state: SyncState::WaitForEvents, - message_count: message_count.clone(), - event_count: event_count.clone(), - tick_count: tick_count.clone(), - }; - - // Create channels - let (_, message_receiver) = mpsc::unbounded_channel(); - let sync_event_sender = broadcast::Sender::::new(100); - let network_event_sender = broadcast::Sender::::new(100); - let (req_tx, _req_rx) = mpsc::unbounded_channel::(); - let requests = RequestSender::new(req_tx); - let shutdown = CancellationToken::new(); - let (progress_sender, _progress_rx) = watch::channel(manager.progress()); - - let context = SyncManagerTaskContext { - message_receiver, - sync_event_sender, - network_event_receiver: network_event_sender.subscribe(), - requests, - shutdown: shutdown.clone(), - progress_sender, - }; - - // Spawn the task using trait's run method - let handle = tokio::spawn(async move { manager.run(context).await }); - - // Let it run for a bit - tokio::time::sleep(Duration::from_millis(250)).await; - - // Signal shutdown - shutdown.cancel(); - - // Wait for task to complete - let result = handle.await.unwrap(); - assert!(result.is_ok()); - - // Verify the returned identifier matches - assert_eq!(result.unwrap(), ManagerIdentifier::BlockHeader); - - // Verify tick was called multiple times - assert!(tick_count.load(Ordering::Relaxed) > 0); - } -} diff --git a/dash-spv/src/test_utils/event_handler.rs b/dash-spv/src/test_utils/event_handler.rs index 7a5dd1143..20651eaad 100644 --- a/dash-spv/src/test_utils/event_handler.rs +++ b/dash-spv/src/test_utils/event_handler.rs @@ -6,7 +6,7 @@ use tokio::sync::{broadcast, watch}; use crate::client::EventHandler; -use crate::network::NetworkEvent; +use crate::network2::NetworkEvent; use crate::sync::{SyncEvent, SyncProgress}; use key_wallet_manager::WalletEvent; diff --git a/dash-spv/src/timer.rs b/dash-spv/src/timer.rs index bf7c8cc0f..5cf9d179c 100644 --- a/dash-spv/src/timer.rs +++ b/dash-spv/src/timer.rs @@ -72,10 +72,34 @@ pub static P_RECV_DATA: Prof = Prof::new("filt.receive_with_data"); pub static P_SEND_PENDING: Prof = Prof::new("filt.send_pending"); pub static P_STORE_MATCH: Prof = Prof::new("filt.store_and_match"); +// Filters TAIL path (runs off the cfilter hot path, driven by BlockProcessed): where the +// gap-limit rescan cascade actually spends its wall clock. SCAN = first pass over a batch's +// filters; BULK = the lookahead match of the derived-script log across all open batches; +// HEAD = the commit head's own catch-up match; DISK = filter writes in the sequential store; +// COMMIT = the whole commit loop. If SCAN+BULK+HEAD << tail duration, we are waiting on the +// matched-block downloads, not on CPU. +pub static P_SCAN: Prof = Prof::new("filt.scan_batch"); +pub static P_BULK: Prof = Prof::new("filt.bulk_rescan"); +pub static P_HEAD: Prof = Prof::new("filt.head_rescan"); +pub static P_DISK: Prof = Prof::new("filt.disk_store"); +pub static P_VERIFY: Prof = Prof::new("filt.verify"); +pub static P_COMMIT: Prof = Prof::new("filt.commit_loop"); + // Arrival path: time to lock the shared Arc> and dispatch each message. // If per-call time balloons with more peers, that Mutex is the ~70k/s serialization point. pub static P_NET_DISPATCH: Prof = Prof::new("net.dispatch(lock)"); +// network2 pump (single task draining PeerEvents): BUSY = per-message dispatch work +// (subscriber lock + clone + channel send); IDLE = time blocked in inbound.recv().await. +// IDLE >> BUSY => the pump is starved (readers/peers slow), not the bottleneck. +pub static P_PUMP_BUSY: Prof = Prof::new("pump.dispatch_busy"); +pub static P_PUMP_IDLE: Prof = Prof::new("pump.idle_wait"); + +// receive_with_data breakdown: FIND = linear scan over batch_trackers to locate +// the height's batch; INSERT = BlockFilter::new (clones bytes) + hashmap inserts. +pub static P_FIND_BATCH: Prof = Prof::new("filt.find_batch"); +pub static P_INSERT_FILTER: Prof = Prof::new("filt.insert_filter"); + // Reader loop (network/manager.rs) per-iteration breakdown. RLOCK+WLOCK are the two peer-lock // acquisitions per message; RECV is the receive_message/select; MSG vs IDLE count how often an // iteration yielded a message vs timed out waiting (IDLE-heavy => peer/network-bound, not reader). @@ -104,7 +128,17 @@ pub fn dump_profile() { &P_RECV_DATA, &P_SEND_PENDING, &P_STORE_MATCH, + &P_SCAN, + &P_BULK, + &P_HEAD, + &P_DISK, + &P_VERIFY, + &P_COMMIT, + &P_FIND_BATCH, + &P_INSERT_FILTER, &P_NET_DISPATCH, + &P_PUMP_BUSY, + &P_PUMP_IDLE, &P_READ_RLOCK, &P_READ_WLOCK, &P_READ_RECV, diff --git a/dash-spv/src/validation/filter.rs b/dash-spv/src/validation/filter.rs index d9a0f8f3f..8d51dcf46 100644 --- a/dash-spv/src/validation/filter.rs +++ b/dash-spv/src/validation/filter.rs @@ -8,7 +8,6 @@ use std::collections::HashMap; use dashcore::bip158::BlockFilter; use dashcore::hash_types::FilterHeader; use key_wallet_manager::FilterMatchKey; -use rayon::prelude::*; use crate::error::{ValidationError, ValidationResult}; use crate::validation::Validator; @@ -69,10 +68,9 @@ impl Validator> for FilterValidator { prev = input.expected_headers[&height]; } - // Verify all filters in parallel let failures: Vec<(u32, String)> = input .filters - .par_iter() + .iter() .filter_map(|(key, filter)| { let height = key.height(); diff --git a/dash-spv/src/validation/header.rs b/dash-spv/src/validation/header.rs index 85903f6ca..aaf758b64 100644 --- a/dash-spv/src/validation/header.rs +++ b/dash-spv/src/validation/header.rs @@ -1,4 +1,3 @@ -use rayon::prelude::*; use std::time::Instant; use crate::error::{ValidationError, ValidationResult}; @@ -18,8 +17,7 @@ impl Validator<&[HashedBlockHeader]> for BlockHeaderValidator { fn validate(&self, hashed_headers: &[HashedBlockHeader]) -> ValidationResult<()> { let start = Instant::now(); - // Check PoW of i and continuity of i-1 to i in parallel - hashed_headers.par_iter().enumerate().try_for_each(|(i, header)| { + for (i, header) in hashed_headers.iter().enumerate() { // For the first header, skip chain continuity check since we don't have i-1 here if i > 0 && header.header().prev_blockhash != *hashed_headers[i - 1].hash() { return Err(ValidationError::InvalidHeaderChain(format!( @@ -32,8 +30,7 @@ impl Validator<&[HashedBlockHeader]> for BlockHeaderValidator { if !header.header().target().is_met_by(*header.hash()) { return Err(ValidationError::InvalidProofOfWork); } - Ok(()) - })?; + } tracing::trace!( "Header chain validation passed for {} headers, duration: {:?}", diff --git a/dash/Cargo.toml b/dash/Cargo.toml index 22d697b9b..8362d666a 100644 --- a/dash/Cargo.toml +++ b/dash/Cargo.toml @@ -34,6 +34,7 @@ quorum_validation = ["bls"] message_verification = ["bls"] bincode = [ "dep:bincode", "dep:bincode_derive", "dashcore_hashes/bincode", "dash-network/bincode" ] test-utils = [] +tokio = ["dep:tokio-util", "dep:bytes"] [package.metadata.docs.rs] all-features = true @@ -57,6 +58,8 @@ bincode = { version = "2.0.1", optional = true } bincode_derive = { version = "2.0.1", optional = true } blsful = { git = "https://github.com/dashpay/agora-blsful", rev = "0c34a7a488a0bd1c9a9a2196e793b303ad35c900", optional = true } ed25519-dalek = { version = "2.1", features = ["rand_core"], optional = true } +tokio-util = { version = "0.7", default-features = false, features = ["codec"], optional = true } +bytes = { version = "1", optional = true } blake3 = "1.8.1" thiserror = "2" bitvec = "1.0" diff --git a/dash/src/network/message.rs b/dash/src/network/message.rs index 140d32d16..d64aaad53 100644 --- a/dash/src/network/message.rs +++ b/dash/src/network/message.rs @@ -666,6 +666,138 @@ impl Decodable for RawNetworkMessage { } } +/// A Tokio [`Decoder`](tokio_util::codec::Decoder)/[`Encoder`](tokio_util::codec::Encoder) +/// that frames one [`RawNetworkMessage`] per message, so a peer connection's read half can +/// be wrapped in `tokio_util::codec::FramedRead` (and the write half in `FramedWrite`) +/// instead of hand-rolling the length-prefixed framing. Enabled by the `tokio` feature. +/// The [`RawNetworkMessage`] decoder validates the payload checksum, so a corrupt +/// frame surfaces as an error rather than a bad message. +#[cfg(feature = "tokio")] +#[derive(Debug, Default, Clone, Copy)] +pub struct RawNetworkMessageCodec; + +#[cfg(feature = "tokio")] +impl tokio_util::codec::Decoder for RawNetworkMessageCodec { + type Item = RawNetworkMessage; + type Error = encode::Error; + + fn decode(&mut self, src: &mut bytes::BytesMut) -> Result, Self::Error> { + use bytes::Buf as _; + + /// Message header: `magic(4) + command(12) + payload_length(4) + checksum(4)`. + const HEADER_LEN: usize = 24; + /// Byte offset of the little-endian payload length within the header. + const LENGTH_OFFSET: usize = 16; + + // Need the whole header before we can read the declared payload length. + if src.len() < HEADER_LEN { + return Ok(None); + } + + let payload_len = u32::from_le_bytes([ + src[LENGTH_OFFSET], + src[LENGTH_OFFSET + 1], + src[LENGTH_OFFSET + 2], + src[LENGTH_OFFSET + 3], + ]) as usize; + + // Reject an absurd declared length *before* reserving buffer for it. + if payload_len > MAX_MSG_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "declared payload length exceeds MAX_MSG_SIZE", + ) + .into()); + } + + let frame_len = HEADER_LEN + payload_len; + if src.len() < frame_len { + src.reserve(frame_len - src.len()); + return Ok(None); + } + + // Decode from the exact frame bytes (a finite, in-memory reader). + let mut cursor = io::Cursor::new(&src[..frame_len]); + let msg = RawNetworkMessage::consensus_decode_from_finite_reader(&mut cursor)?; + src.advance(frame_len); + + Ok(Some(msg)) + } +} + +#[cfg(feature = "tokio")] +impl tokio_util::codec::Encoder<&RawNetworkMessage> for RawNetworkMessageCodec { + type Error = encode::Error; + + fn encode( + &mut self, + item: &RawNetworkMessage, + dst: &mut bytes::BytesMut, + ) -> Result<(), Self::Error> { + dst.extend_from_slice(&serialize(item)); + Ok(()) + } +} + +#[cfg(feature = "tokio")] +impl tokio_util::codec::Encoder for RawNetworkMessageCodec { + type Error = encode::Error; + + fn encode( + &mut self, + item: RawNetworkMessage, + dst: &mut bytes::BytesMut, + ) -> Result<(), Self::Error> { + >::encode(self, &item, dst) + } +} + +#[cfg(all(test, feature = "tokio"))] +mod codec_tests { + use super::{NetworkMessage, RawNetworkMessage, RawNetworkMessageCodec}; + use bytes::BytesMut; + use tokio_util::codec::{Decoder, Encoder}; + + #[test] + fn roundtrip_frame() { + let msg = RawNetworkMessage { + magic: 0xBD6B_0CBF, + payload: NetworkMessage::Ping(0x0102_0304_0506_0708), + }; + let mut codec = RawNetworkMessageCodec; + let mut buf = BytesMut::new(); + codec.encode(&msg, &mut buf).unwrap(); + + let decoded = codec.decode(&mut buf).unwrap().expect("full frame decodes"); + assert_eq!(decoded.magic, msg.magic); + assert!(matches!(decoded.payload, NetworkMessage::Ping(0x0102_0304_0506_0708))); + assert!(buf.is_empty()); + } + + #[test] + fn partial_frame_yields_none_until_complete() { + let msg = RawNetworkMessage { + magic: 0xBD6B_0CBF, + payload: NetworkMessage::Ping(42), + }; + let mut codec = RawNetworkMessageCodec; + let mut full = BytesMut::new(); + codec.encode(&msg, &mut full).unwrap(); + + let mut partial = BytesMut::new(); + for (i, byte) in full.iter().enumerate() { + partial.extend_from_slice(&[*byte]); + let out = codec.decode(&mut partial).unwrap(); + if i + 1 < full.len() { + assert!(out.is_none()); + } else { + assert!(out.is_some()); + assert!(partial.is_empty()); + } + } + } +} + #[cfg(test)] mod test { use std::net::Ipv4Addr; From 55b40e01347c66b03c7e44761c66d50467570946 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Wed, 15 Jul 2026 23:47:55 +0000 Subject: [PATCH 03/25] wiring --- dash-spv-ffi/src/callbacks.rs | 20 +- dash-spv-ffi/src/client.rs | 18 +- dash-spv-ffi/tests/test_types.rs | 6 +- dash-spv/examples/filter_sync.rs | 7 +- dash-spv/examples/simple_sync.rs | 7 +- dash-spv/examples/spv_with_wallet.rs | 7 +- dash-spv/src/client/core.rs | 14 +- dash-spv/src/client/event_handler.rs | 9 +- dash-spv/src/client/events.rs | 2 +- dash-spv/src/client/lifecycle.rs | 80 +- dash-spv/src/lib.rs | 6 +- dash-spv/src/main.rs | 42 +- dash-spv/src/network/addrv2.rs | 236 -- dash-spv/src/network/constants.rs | 26 - dash-spv/src/network/discovery.rs | 180 +- dash-spv/src/network/event.rs | 62 - dash-spv/src/network/handshake.rs | 320 --- dash-spv/src/network/manager.rs | 2478 +++++++---------- dash-spv/src/network/message_dispatcher.rs | 227 -- dash-spv/src/network/message_type.rs | 170 -- dash-spv/src/network/mod.rs | 273 +- dash-spv/src/network/peer.rs | 1230 +++----- dash-spv/src/network/pool.rs | 316 --- dash-spv/src/network/reputation.rs | 439 --- dash-spv/src/network/reputation_tests.rs | 109 - dash-spv/src/network/tests.rs | 119 - dash-spv/src/network2/discovery.rs | 71 - dash-spv/src/network2/manager.rs | 1036 ------- dash-spv/src/network2/mod.rs | 5 - dash-spv/src/network2/peer.rs | 536 ---- dash-spv/src/storage/peers.rs | 46 +- dash-spv/src/storage/segments.rs | 82 +- dash-spv/src/sync/block_headers/manager.rs | 2 +- dash-spv/src/sync/block_headers/pipeline.rs | 24 +- .../src/sync/block_headers/segment_state.rs | 15 +- .../src/sync/block_headers/sync_manager.rs | 10 +- dash-spv/src/sync/blocks/manager.rs | 22 +- dash-spv/src/sync/blocks/pipeline.rs | 17 +- dash-spv/src/sync/blocks/sync_manager.rs | 16 +- dash-spv/src/sync/chainlock/sync_manager.rs | 8 +- dash-spv/src/sync/filter_headers/manager.rs | 2 +- dash-spv/src/sync/filter_headers/pipeline.rs | 2 +- .../src/sync/filter_headers/sync_manager.rs | 12 +- dash-spv/src/sync/filters/manager.rs | 764 ++++- dash-spv/src/sync/filters/pipeline.rs | 2 +- dash-spv/src/sync/filters/sync_manager.rs | 36 +- dash-spv/src/sync/instantsend/sync_manager.rs | 8 +- dash-spv/src/sync/masternodes/manager.rs | 2 +- dash-spv/src/sync/masternodes/pipeline.rs | 2 +- dash-spv/src/sync/masternodes/sync_manager.rs | 10 +- dash-spv/src/sync/mempool/manager.rs | 26 +- dash-spv/src/sync/mempool/sync_manager.rs | 39 +- dash-spv/src/sync/sync_coordinator.rs | 4 +- dash-spv/src/sync/sync_manager.rs | 91 +- dash-spv/src/test_utils/event_handler.rs | 2 +- dash-spv/src/test_utils/mod.rs | 2 - dash-spv/src/test_utils/network.rs | 193 -- dash-spv/src/timer.rs | 2 +- dash-spv/tests/dashd_masternode/setup.rs | 17 +- dash-spv/tests/dashd_sync/helpers.rs | 39 + dash-spv/tests/dashd_sync/setup.rs | 13 +- dash-spv/tests/dashd_sync/tests_mempool.rs | 8 +- .../tests/dashd_sync/tests_multi_wallet.rs | 74 +- .../tests/dashd_sync/tests_transaction.rs | 95 +- dash-spv/tests/peer_test.rs | 232 -- dash-spv/tests/test_handshake_logic.rs | 16 - dash-spv/tests/wallet_integration_test.rs | 10 +- dash/src/network/message.rs | 46 +- dash/src/network/message_sml.rs | 4 +- masternode-seeds-fetcher/Cargo.toml | 3 +- masternode-seeds-fetcher/src/main.rs | 5 +- masternode-seeds-fetcher/src/peer.rs | 74 + masternode-seeds-fetcher/src/probe.rs | 2 +- 73 files changed, 2934 insertions(+), 7196 deletions(-) delete mode 100644 dash-spv/src/network/addrv2.rs delete mode 100644 dash-spv/src/network/constants.rs delete mode 100644 dash-spv/src/network/event.rs delete mode 100644 dash-spv/src/network/handshake.rs delete mode 100644 dash-spv/src/network/message_dispatcher.rs delete mode 100644 dash-spv/src/network/message_type.rs delete mode 100644 dash-spv/src/network/pool.rs delete mode 100644 dash-spv/src/network/reputation.rs delete mode 100644 dash-spv/src/network/reputation_tests.rs delete mode 100644 dash-spv/src/network/tests.rs delete mode 100644 dash-spv/src/network2/discovery.rs delete mode 100644 dash-spv/src/network2/manager.rs delete mode 100644 dash-spv/src/network2/mod.rs delete mode 100644 dash-spv/src/network2/peer.rs delete mode 100644 dash-spv/src/test_utils/network.rs delete mode 100644 dash-spv/tests/peer_test.rs delete mode 100644 dash-spv/tests/test_handshake_logic.rs create mode 100644 masternode-seeds-fetcher/src/peer.rs diff --git a/dash-spv-ffi/src/callbacks.rs b/dash-spv-ffi/src/callbacks.rs index ff1d1c765..1f9954e7c 100644 --- a/dash-spv-ffi/src/callbacks.rs +++ b/dash-spv-ffi/src/callbacks.rs @@ -318,6 +318,7 @@ impl FFISyncEventCallbacks { start_height, end_height, tip_height, + .. } => { if let Some(cb) = self.on_filter_headers_stored { cb(*start_height, *end_height, *tip_height, self.user_data); @@ -438,6 +439,11 @@ impl FFISyncEventCallbacks { cb(*header_tip, *cycle, self.user_data); } } + // Internal hand-off between the header and filter-header pipelines (headers + // are passed in memory before they are persisted). Nothing to surface. + SyncEvent::BlockHeadersInMemory { + .. + } => {} } } } @@ -503,17 +509,13 @@ impl FFINetworkEventCallbacks { use dash_spv::network::NetworkEvent; match event { - NetworkEvent::PeerConnected { - address, - } => { + NetworkEvent::PeerConnected(address) => { if let Some(cb) = self.on_peer_connected { let c_addr = CString::new(address.to_string()).unwrap_or_default(); cb(c_addr.as_ptr(), self.user_data); } } - NetworkEvent::PeerDisconnected { - address, - } => { + NetworkEvent::PeerDisconnected(address) => { if let Some(cb) = self.on_peer_disconnected { let c_addr = CString::new(address.to_string()).unwrap_or_default(); cb(c_addr.as_ptr(), self.user_data); @@ -522,12 +524,14 @@ impl FFINetworkEventCallbacks { NetworkEvent::PeersUpdated { connected_count, best_height, - .. } => { if let Some(cb) = self.on_peers_updated { - cb(*connected_count as u32, best_height.unwrap_or(0), self.user_data); + cb(*connected_count, *best_height, self.user_data); } } + // Router bookkeeping: which request just went on the wire. Internal to the + // sync pipelines, nothing an FFI consumer can act on. + NetworkEvent::RequestOnFlight(_) => {} } } } diff --git a/dash-spv-ffi/src/client.rs b/dash-spv-ffi/src/client.rs index 6cb94c8e2..d9335eea1 100644 --- a/dash-spv-ffi/src/client.rs +++ b/dash-spv-ffi/src/client.rs @@ -17,7 +17,6 @@ use tokio::task::JoinHandle; /// FFI wrapper around `DashSpvClient`. type InnerClient = DashSpvClient< key_wallet_manager::WalletManager, - dash_spv::network::PeerNetworkManager, DiskStorageManager, >; @@ -82,26 +81,17 @@ pub unsafe extern "C" fn dash_spv_ffi_client_new( let client_result = runtime.block_on(async move { // Construct concrete implementations for generics - let network = dash_spv::network::PeerNetworkManager::new(&client_config).await; let storage = DiskStorageManager::new(&client_config).await; let wallet = key_wallet_manager::WalletManager::< key_wallet::wallet::managed_wallet_info::ManagedWalletInfo, >::new(client_config.network); let wallet = std::sync::Arc::new(tokio::sync::RwLock::new(wallet)); - match (network, storage) { - (Ok(network), Ok(storage)) => { - DashSpvClient::new( - client_config, - network, - storage, - wallet, - vec![Arc::new(callbacks)], - ) - .await + match storage { + Ok(storage) => { + DashSpvClient::new(client_config, storage, wallet, vec![Arc::new(callbacks)]).await } - (Err(e), _) => Err(e), - (_, Err(e)) => Err(dash_spv::SpvError::Storage(e)), + Err(e) => Err(dash_spv::SpvError::Storage(e)), } }); diff --git a/dash-spv-ffi/tests/test_types.rs b/dash-spv-ffi/tests/test_types.rs index 4b807fa9a..58b8dbc5e 100644 --- a/dash-spv-ffi/tests/test_types.rs +++ b/dash-spv-ffi/tests/test_types.rs @@ -109,7 +109,11 @@ mod tests { let ffi_progress = FFISyncProgress::from(progress); assert_eq!(ffi_progress.state, FFISyncState::Syncing); - assert_eq!(ffi_progress.percentage, 0.625); + assert!( + (ffi_progress.percentage - 0.675).abs() < 1e-9, + "percentage was {}", + ffi_progress.percentage + ); // Verify headers progress assert!(!ffi_progress.headers.is_null()); diff --git a/dash-spv/examples/filter_sync.rs b/dash-spv/examples/filter_sync.rs index 203896ce6..fdd5c38d9 100644 --- a/dash-spv/examples/filter_sync.rs +++ b/dash-spv/examples/filter_sync.rs @@ -1,6 +1,5 @@ //! BIP157 filter synchronization example. -use dash_spv::network::PeerNetworkManager; use dash_spv::storage::DiskStorageManager; use dash_spv::{init_console_logging, ClientConfig, DashSpvClient, LevelFilter}; use dashcore::Address; @@ -25,9 +24,6 @@ async fn main() -> Result<(), Box> { .with_storage_path("./.tmp/filter-sync-example-storage") .without_masternodes(); // Skip masternode sync for this example - // Create network manager - let network_manager = PeerNetworkManager::new(&config).await?; - // Create storage manager let storage_manager = DiskStorageManager::new(&config).await?; @@ -35,8 +31,7 @@ async fn main() -> Result<(), Box> { let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); // Create the client - let client = - DashSpvClient::new(config, network_manager, storage_manager, wallet, vec![]).await?; + let client = DashSpvClient::new(config, storage_manager, wallet, vec![]).await?; println!("Starting synchronization with filter support..."); println!("Watching address: {:?}", watch_address); diff --git a/dash-spv/examples/simple_sync.rs b/dash-spv/examples/simple_sync.rs index 0568768fe..ac24abb22 100644 --- a/dash-spv/examples/simple_sync.rs +++ b/dash-spv/examples/simple_sync.rs @@ -1,6 +1,5 @@ //! Simple header synchronization example. -use dash_spv::network::PeerNetworkManager; use dash_spv::storage::DiskStorageManager; use dash_spv::{init_console_logging, ClientConfig, DashSpvClient, LevelFilter}; use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; @@ -20,9 +19,6 @@ async fn main() -> Result<(), Box> { .without_filters() // Skip filter sync for this example .without_masternodes(); // Skip masternode sync for this example - // Create network manager - let network_manager = PeerNetworkManager::new(&config).await?; - // Create storage manager let storage_manager = DiskStorageManager::new(&config).await?; @@ -30,8 +26,7 @@ async fn main() -> Result<(), Box> { let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); // Create the client - let client = - DashSpvClient::new(config, network_manager, storage_manager, wallet, vec![]).await?; + let client = DashSpvClient::new(config, storage_manager, wallet, vec![]).await?; println!("Starting header synchronization..."); diff --git a/dash-spv/examples/spv_with_wallet.rs b/dash-spv/examples/spv_with_wallet.rs index 2d2c661d8..93655db95 100644 --- a/dash-spv/examples/spv_with_wallet.rs +++ b/dash-spv/examples/spv_with_wallet.rs @@ -2,7 +2,6 @@ //! //! This example shows how to integrate the SPV client with a wallet manager. -use dash_spv::network::PeerNetworkManager; use dash_spv::storage::DiskStorageManager; use dash_spv::{ClientConfig, DashSpvClient, LevelFilter}; use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; @@ -20,9 +19,6 @@ async fn main() -> Result<(), Box> { .with_storage_path("./.tmp/spv-with-wallet-example-storage") .with_validation_mode(dash_spv::ValidationMode::Full); - // Create network manager - let network_manager = PeerNetworkManager::new(&config).await?; - // Create storage manager - use disk storage for persistence let storage_manager = DiskStorageManager::new(&config).await?; @@ -30,8 +26,7 @@ async fn main() -> Result<(), Box> { let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); // Create the SPV client with all components - let client = - DashSpvClient::new(config, network_manager, storage_manager, wallet, vec![]).await?; + let client = DashSpvClient::new(config, storage_manager, wallet, vec![]).await?; // The wallet will automatically be notified of: // - New blocks via process_block() diff --git a/dash-spv/src/client/core.rs b/dash-spv/src/client/core.rs index c8c1b714b..63251f187 100644 --- a/dash-spv/src/client/core.rs +++ b/dash-spv/src/client/core.rs @@ -56,8 +56,7 @@ pub(super) type PersistentSyncCoordinator = SyncCoordinator< /// - Essential for a reusable library /// /// ### 4. **Testing Without Mocks** 🧪 -/// - Test implementations (`MockNetworkManager`) are -/// first-class types, not runtime injections +/// - Test implementations are first-class types, not runtime injections /// - No conditional compilation or feature flags needed for tests /// - Type system ensures test and production code are compatible /// @@ -70,7 +69,7 @@ pub(super) type PersistentSyncCoordinator = SyncCoordinator< /// /// - `W: WalletInterface` - Handles UTXO tracking, address management, transaction processing /// - `S: StorageManager` - Persistent storage for headers, filters, chain state -/// - Networking is the concrete `crate::network2::PeerNetworkManager` +/// - Networking is the concrete `crate::network::PeerNetworkManager` /// - Event handlers are stored as `Vec>` /// /// ## Common Configurations @@ -96,7 +95,7 @@ pub(super) type PersistentSyncCoordinator = SyncCoordinator< /// The generic design is an intentional, beneficial architectural choice for a library. pub struct DashSpvClient { pub(super) config: Arc>, - pub(super) network: Arc, + pub(super) network: Arc, pub(super) storage: Arc>, /// External wallet implementation (required) pub(super) wallet: Arc>, @@ -105,6 +104,12 @@ pub struct DashSpvClient { /// `true` while running, `false` once a stop is requested. Stored as a /// `watch` so a stop is observed immediately rather than polled. pub(super) running: Arc>, + /// Set by `stop()` before it flips `running` false; read by `start()` under + /// the `running` watch lock so a stop that races startup is not lost. Without + /// it, a `stop()` arriving while `start()` is still connecting would no-op + /// (running not yet true), then `start()` would flip running true and the run + /// task would sync forever — hanging any `run_handle.await`. + pub(super) stop_requested: Arc, pub(super) event_handlers: Arc>>, } @@ -118,6 +123,7 @@ impl Clone for DashSpvClient { masternode_engine: self.masternode_engine.clone(), sync_coordinator: Arc::clone(&self.sync_coordinator), running: Arc::clone(&self.running), + stop_requested: Arc::clone(&self.stop_requested), event_handlers: Arc::clone(&self.event_handlers), } } diff --git a/dash-spv/src/client/event_handler.rs b/dash-spv/src/client/event_handler.rs index bc5264e6a..a8ecb67cf 100644 --- a/dash-spv/src/client/event_handler.rs +++ b/dash-spv/src/client/event_handler.rs @@ -11,7 +11,7 @@ use tokio::sync::{broadcast, mpsc, watch, RwLock}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; -use crate::network2::NetworkEvent; +use crate::network::NetworkEvent; use crate::sync::{SyncEvent, SyncProgress}; use dashcore::ephemerealdata::chain_lock::ChainLock; use key_wallet_manager::{WalletEvent, WalletInterface}; @@ -227,7 +227,7 @@ mod tests { use tokio_util::sync::CancellationToken; use super::{spawn_broadcast_monitor, spawn_progress_monitor, EventHandler}; - use crate::network2::NetworkEvent; + use crate::network::NetworkEvent; use crate::sync::{ManagerIdentifier, SyncEvent, SyncProgress}; use key_wallet_manager::WalletEvent; @@ -282,7 +282,10 @@ mod tests { tip_height: 100, }; handler.on_sync_event(&event); - handler.on_network_event(&NetworkEvent::PeersUpdated); + handler.on_network_event(&NetworkEvent::PeersUpdated { + connected_count: 1, + best_height: 100, + }); handler.on_progress(&SyncProgress::default()); handler.on_error("test error"); } diff --git a/dash-spv/src/client/events.rs b/dash-spv/src/client/events.rs index 7da9abb93..19b692c6d 100644 --- a/dash-spv/src/client/events.rs +++ b/dash-spv/src/client/events.rs @@ -6,7 +6,7 @@ use tokio::sync::watch; -use crate::network2::NetworkEvent; +use crate::network::NetworkEvent; use crate::storage::StorageManager; use crate::sync::{SyncEvent, SyncProgress}; use key_wallet_manager::WalletInterface; diff --git a/dash-spv/src/client/lifecycle.rs b/dash-spv/src/client/lifecycle.rs index 65aea96e1..f30729a01 100644 --- a/dash-spv/src/client/lifecycle.rs +++ b/dash-spv/src/client/lifecycle.rs @@ -31,10 +31,12 @@ use std::sync::Arc; use tokio::sync::{watch, Mutex, RwLock}; impl DashSpvClient { - /// Create a new SPV client with the given configuration, network, storage, and wallet. + /// Create a new SPV client with the given configuration, storage, and wallet. + /// + /// The peer-to-peer network layer is an internal implementation detail: the + /// client builds and owns it from `config`, so it is not part of the public API. pub async fn new( config: ClientConfig, - network: crate::network2::PeerNetworkManager, mut storage: S, wallet: Arc>, event_handlers: Vec>, @@ -45,6 +47,10 @@ impl DashSpvClient { config.validate().map_err(SpvError::Config)?; config.apply_global_overrides().map_err(SpvError::Config)?; + // The network manager is built here rather than passed in: it is a concrete, + // internal type and keeping it out of the constructor keeps it off the public API. + let network = crate::network::PeerNetworkManager::new(&config).await; + // Resolve where to anchor the chain. An explicit `start_from_height` always // wins. Otherwise fall back to the wallet birth height so we don't sync headers // and filter headers from genesis when the wallet only cares about recent blocks. @@ -100,12 +106,11 @@ impl DashSpvClient { storage.block_headers(), storage.filter_headers(), storage.filters(), + storage.metadata(), ) .await, ); - managers.blocks = Some( - BlocksManager::new(wallet.clone(), storage.block_headers(), storage.blocks()).await, - ); + managers.blocks = Some(BlocksManager::new(wallet.clone(), storage.blocks()).await); } // Build masternode manager if enabled @@ -156,6 +161,7 @@ impl DashSpvClient { masternode_engine, sync_coordinator: Arc::new(Mutex::new(sync_coordinator)), running: Arc::new(watch::Sender::new(false)), + stop_requested: Arc::new(std::sync::atomic::AtomicBool::new(false)), event_handlers: Arc::new(event_handlers), }; @@ -177,29 +183,52 @@ impl DashSpvClient { return Err(SpvError::Config("Client already running".to_string())); } - // Start all sync tasks before connecting to the network to make sure initial connection - // events are handled correctly in the sync coordinator. - // network2's `PeerNetworkManager` connects during its `new()`, so there is no - // separate connect step here. + // Fresh lifecycle: clear any stop request left over from a previous run so + // this start can complete. A `stop()` arriving *after* this point (while we + // connect below) is recorded and honored by the guarded transition at the end. + self.stop_requested.store(false, std::sync::atomic::Ordering::SeqCst); + + // Spawn every sync task BEFORE connecting, so each manager has subscribed by the + // time the network announces its peers. The network manager used to connect inside + // `new()`, which meant `PeersUpdated` and every `PeerConnected` fired into the void + // — and the mempool manager, whose peer set is built from those events, stayed empty + // and never sent the `filterload` that enables transaction relay. if let Err(e) = self.sync_coordinator.lock().await.start(&self.network).await { tracing::error!("Failed to start sync coordinator: {}", e); return Err(SpvError::Sync(e)); } - // Only mark as running after all startup operations succeed. - // `send_replace` always stores the value regardless of receiver count, - // so this is correct even when `run()` has not subscribed yet. - self.running.send_replace(true); + self.network.start().await; + + // Only mark as running after all startup operations succeed — and only if + // no `stop()` raced in while we were connecting. The check runs inside the + // watch lock (via `send_if_modified`), and `stop()` sets `stop_requested` + // before it flips `running`, so the two orderings are both safe: + // - we win the lock first: set running=true; a later stop() flips it false. + // - stop() won: `stop_requested` is already true here, so we leave running + // false and the run loop tears down immediately instead of syncing forever. + // `send_if_modified` stores the value regardless of receiver count, so this + // is correct even when `run()` has not subscribed yet. + self.running.send_if_modified(|running| { + if self.stop_requested.load(std::sync::atomic::Ordering::SeqCst) { + false + } else { + *running = true; + true + } + }); Ok(()) } /// Stop the SPV client. pub async fn stop(&self) -> Result<()> { - // Check if already stopped - if !*self.running.borrow() { - return Ok(()); - } + // Record the stop request BEFORE flipping `running`, so a `start()` still + // connecting observes it (under the watch lock) and declines to mark the + // client running. Otherwise a stop that arrives mid-startup would be lost: + // `start()` would flip running true afterwards and the run task would sync + // forever, hanging `run_handle.await`. + self.stop_requested.store(true, std::sync::atomic::Ordering::SeqCst); // Flip the running state before tearing anything down so a concurrent // `run()` loop wakes immediately and breaks out before it can lock the @@ -207,6 +236,14 @@ impl DashSpvClient { // shutdown below. self.running.send_replace(false); + // Always tear down, even if `running` was never true: a stop that raced + // startup leaves `running` false yet `start()` may already have spawned + // the coordinator's manager tasks and connected the network, so those must + // still be stopped. Every step below is idempotent (cancelling an already + // cancelled token, draining an empty task set, persisting again), so the + // normal double call — this stop() plus the run loop's own final stop() — + // is safe. + // Shut down sync coordinator: signals cancellation and waits for manager // tasks to drain before we tear down the network and storage layers. if let Err(e) = self.sync_coordinator.lock().await.shutdown().await { @@ -341,14 +378,9 @@ impl DashSpvClient { /// Load wallet data from storage. pub(super) async fn load_wallet_data(&self) -> Result<()> { - tracing::info!("Loading wallet data from storage..."); - - let _wallet = self.wallet.read().await; - - // The wallet implementation is responsible for managing its own persistent state - // The SPV client will notify it of new blocks/transactions through the WalletInterface + // The wallet implementation is responsible for managing its own persistent state; + // the SPV client notifies it of new blocks/transactions through the WalletInterface. tracing::info!("Wallet data loading is handled by the wallet implementation"); - Ok(()) } } diff --git a/dash-spv/src/lib.rs b/dash-spv/src/lib.rs index a5cee8c80..dcd9ec5e4 100644 --- a/dash-spv/src/lib.rs +++ b/dash-spv/src/lib.rs @@ -13,7 +13,6 @@ //! //! ```no_run //! use dash_spv::{DashSpvClient, ClientConfig}; -//! use dash_spv::network::PeerNetworkManager; //! use dash_spv::storage::DiskStorageManager; //! use dashcore::Network; //! use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; @@ -27,15 +26,13 @@ //! let config = ClientConfig::mainnet() //! .with_storage_path("./.tmp/example-storage"); //! -//! // Create the required components -//! let network = PeerNetworkManager::new(&config).await?; +//! // Create the required components. //! let storage = DiskStorageManager::new(&config).await?; //! let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); //! //! // Create and run the client //! let client = DashSpvClient::new( //! config.clone(), -//! network, //! storage, //! wallet, //! vec![Arc::new(())], @@ -67,7 +64,6 @@ pub mod client; pub mod error; pub mod logging; pub mod network; -pub mod network2; pub mod storage; pub mod sync; pub mod timer; diff --git a/dash-spv/src/main.rs b/dash-spv/src/main.rs index eca7d7a6a..69a029e2d 100644 --- a/dash-spv/src/main.rs +++ b/dash-spv/src/main.rs @@ -258,15 +258,6 @@ async fn run() -> Result<(), Box> { )?; let wallet = Arc::new(tokio::sync::RwLock::new(wallet_manager)); - // Create network manager - let network_manager = match dash_spv::network::manager::PeerNetworkManager::new(&config).await { - Ok(nm) => nm, - Err(e) => { - eprintln!("Failed to create network manager: {}", e); - process::exit(1); - } - }; - let storage_manager = match dash_spv::storage::DiskStorageManager::new(&config).await { Ok(sm) => sm, Err(e) => { @@ -274,7 +265,7 @@ async fn run() -> Result<(), Box> { process::exit(1); } }; - run_client(config, network_manager, storage_manager, wallet).await?; + run_client(config, storage_manager, wallet).await?; Ok(()) } @@ -382,27 +373,24 @@ fn parse_llmq_devnet_params(raw: &str) -> Result { async fn run_client( config: ClientConfig, - network_manager: dash_spv::network::manager::PeerNetworkManager, storage_manager: S, wallet: Arc>>, ) -> Result<(), Box> { // Create and start the client - let client = - match DashSpvClient::< - WalletManager, - dash_spv::network::manager::PeerNetworkManager, - S, - >::new( - config.clone(), network_manager, storage_manager, wallet.clone(), Vec::new() - ) - .await - { - Ok(client) => client, - Err(e) => { - eprintln!("Failed to create SPV client: {}", e); - process::exit(1); - } - }; + let client = match DashSpvClient::, S>::new( + config.clone(), + storage_manager, + wallet.clone(), + Vec::new(), + ) + .await + { + Ok(client) => client, + Err(e) => { + eprintln!("Failed to create SPV client: {}", e); + process::exit(1); + } + }; let stop_client = client.clone(); tokio::spawn(async move { diff --git a/dash-spv/src/network/addrv2.rs b/dash-spv/src/network/addrv2.rs deleted file mode 100644 index 839f8a0d9..000000000 --- a/dash-spv/src/network/addrv2.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! AddrV2 message handling for modern peer exchange protocol - -use rand::prelude::*; -use std::collections::{HashMap, HashSet}; -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use tokio::sync::RwLock; - -use dashcore::network::address::{AddrV2, AddrV2Message}; -use dashcore::network::constants::ServiceFlags; -use dashcore::network::message::NetworkMessage; - -use crate::network::constants::{MAX_ADDR_TO_SEND, MAX_ADDR_TO_STORE}; - -const ONE_WEEK: u32 = 7 * 24 * 60 * 60; -const TEN_MINUTES: u32 = 600; - -/// Evict oldest entries if the map exceeds capacity, keeping the freshest addresses. -fn evict_if_needed(peers: &mut HashMap) { - if peers.len() > MAX_ADDR_TO_STORE { - let mut entries: Vec<_> = peers.drain().collect(); - entries.sort_by_key(|(_, msg)| std::cmp::Reverse(msg.time)); - entries.truncate(MAX_ADDR_TO_STORE); - peers.extend(entries); - } -} - -/// Handler for AddrV2 peer exchange protocol -pub struct AddrV2Handler { - /// Known peer addresses from AddrV2 messages - known_peers: Arc>>, - /// Peers that support AddrV2 - supports_addrv2: Arc>>, -} - -impl AddrV2Handler { - /// Create a new AddrV2 handler - pub fn new() -> Self { - Self { - known_peers: Arc::new(RwLock::new(HashMap::new())), - supports_addrv2: Arc::new(RwLock::new(HashSet::new())), - } - } - - /// Handle SendAddrV2 message indicating peer support - pub async fn handle_sendaddrv2(&self, peer_addr: SocketAddr) { - self.supports_addrv2.write().await.insert(peer_addr); - tracing::debug!("Peer {} supports AddrV2", peer_addr); - } - - /// Handle incoming AddrV2 messages - pub async fn handle_addrv2(&self, messages: Vec) { - let mut known_peers = self.known_peers.write().await; - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|e| { - tracing::error!("System time error in handle_addrv2: {}", e); - Duration::from_secs(0) - }) - .as_secs() as u32; - - let received = messages.len(); - let mut added = 0; - let mut updated = 0; - - for msg in messages { - // Accept addresses seen within the last week. Older addresses are likely stale. - // Also, reject timestamps more than 10 minutes in the future which are invalid. - if msg.time < now.saturating_sub(ONE_WEEK) || msg.time > now + TEN_MINUTES { - tracing::trace!("Ignoring AddrV2 with invalid timestamp: {}", msg.time); - continue; - } - - let Ok(socket_addr) = msg.socket_addr() else { - continue; - }; - - // Only update if new or has fresher timestamp - match known_peers.get(&socket_addr) { - Some(existing) if existing.time >= msg.time => continue, - Some(_) => updated += 1, - None => added += 1, - } - known_peers.insert(socket_addr, msg); - } - - evict_if_needed(&mut known_peers); - - tracing::info!( - "Processed AddrV2 messages: received {}, added {}, updated {}, total known peers: {}", - received, - added, - updated, - known_peers.len() - ); - } - - /// Get addresses to share with a peer - pub async fn get_addresses_for_peer(&self, count: usize) -> Vec { - let known_peers = self.known_peers.read().await; - - if known_peers.is_empty() { - return vec![]; - } - - // Select random subset - let mut rng = thread_rng(); - let count = count.min(MAX_ADDR_TO_SEND).min(known_peers.len()); - - let addresses: Vec = - known_peers.values().choose_multiple(&mut rng, count).into_iter().cloned().collect(); - - addresses - } - - /// Check if a peer supports AddrV2 - pub async fn peer_supports_addrv2(&self, addr: &SocketAddr) -> bool { - self.supports_addrv2.read().await.contains(addr) - } - - /// Get all known socket addresses - pub async fn get_known_addresses(&self) -> Vec { - self.known_peers.read().await.values().cloned().collect() - } - - /// Add a known peer address - pub async fn add_known_address(&self, addr: SocketAddr, services: ServiceFlags) { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_else(|e| { - tracing::error!("System time error in add_known_address: {}", e); - Duration::from_secs(0) - }) - .as_secs() as u32; - - let addr_v2 = match addr.ip() { - std::net::IpAddr::V4(ipv4) => AddrV2::Ipv4(ipv4), - std::net::IpAddr::V6(ipv6) => AddrV2::Ipv6(ipv6), - }; - - let addr_msg = AddrV2Message { - time: now, - services, - addr: addr_v2, - port: addr.port(), - }; - - let mut known_peers = self.known_peers.write().await; - known_peers.insert(addr, addr_msg); - evict_if_needed(&mut known_peers); - } - - /// Build a GetAddr response message - pub async fn build_addr_response(&self) -> NetworkMessage { - let addresses = self.get_addresses_for_peer(23).await; // Bitcoin typically sends ~23 addresses - NetworkMessage::AddrV2(addresses) - } -} - -impl Default for AddrV2Handler { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use dashcore::network::address::AddrV2; - - #[tokio::test] - async fn test_addrv2_handler_basic() { - let handler = AddrV2Handler::new(); - - // Test SendAddrV2 support tracking - let peer = "127.0.0.1:9999".parse().expect("Failed to parse test peer address"); - handler.handle_sendaddrv2(peer).await; - assert!(handler.peer_supports_addrv2(&peer).await); - - // Test adding known address - let addr = "192.168.1.1:9999".parse().expect("Failed to parse test address"); - handler.add_known_address(addr, ServiceFlags::NETWORK).await; - - let known = handler.get_known_addresses().await; - assert_eq!(known.len(), 1); - assert_eq!(known[0].socket_addr().unwrap(), addr); - } - - #[tokio::test] - async fn test_addrv2_timestamp_validation() { - let handler = AddrV2Handler::new(); - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("Failed to get system time in test") - .as_secs() as u32; - - // Create test messages with various timestamps - let addr: SocketAddr = - "127.0.0.1:9999".parse().expect("Failed to parse test socket address"); - let ipv4_addr = match addr.ip() { - std::net::IpAddr::V4(v4) => v4, - _ => panic!("Test expects IPv4 address but got IPv6"), - }; - - let messages = vec![ - // Valid: current time - AddrV2Message { - time: now, - services: ServiceFlags::NETWORK, - addr: AddrV2::Ipv4(ipv4_addr), - port: addr.port(), - }, - // Invalid: too old (4 hours ago) - AddrV2Message { - time: now.saturating_sub(14400), - services: ServiceFlags::NETWORK, - addr: AddrV2::Ipv4(ipv4_addr), - port: addr.port(), - }, - // Invalid: too far in future (20 minutes) - AddrV2Message { - time: now + 1200, - services: ServiceFlags::NETWORK, - addr: AddrV2::Ipv4(ipv4_addr), - port: addr.port(), - }, - ]; - - handler.handle_addrv2(messages).await; - - // Only the valid message should be stored - let known = handler.get_known_addresses().await; - assert_eq!(known.len(), 1); - } -} diff --git a/dash-spv/src/network/constants.rs b/dash-spv/src/network/constants.rs deleted file mode 100644 index 30928f70e..000000000 --- a/dash-spv/src/network/constants.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Network constants for peer support - -use std::time::Duration; - -// Timeouts -pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(30); -pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); -pub const MESSAGE_TIMEOUT: Duration = Duration::from_secs(120); -pub const PING_INTERVAL: Duration = Duration::from_secs(120); - -// Reconnection -pub const RECONNECT_DELAY: Duration = Duration::from_secs(5); -pub const MAX_RECONNECT_ATTEMPTS: u32 = 3; - -// Peer exchange -pub const MAX_ADDR_TO_SEND: usize = 1000; -pub const MAX_ADDR_TO_STORE: usize = 2000; - -// Connection maintenance -pub const MAINTENANCE_INTERVAL: Duration = Duration::from_secs(10); // Check more frequently -pub const PEER_DISCOVERY_INTERVAL: Duration = Duration::from_secs(60); // Discover more frequently - -// DNS and polling intervals -pub const DNS_DISCOVERY_DELAY: Duration = Duration::from_secs(10); -pub const MESSAGE_POLL_INTERVAL: Duration = Duration::from_millis(10); -pub const MESSAGE_RECEIVE_TIMEOUT: Duration = Duration::from_millis(100); diff --git a/dash-spv/src/network/discovery.rs b/dash-spv/src/network/discovery.rs index 035bc7f28..aa335053d 100644 --- a/dash-spv/src/network/discovery.rs +++ b/dash-spv/src/network/discovery.rs @@ -1,50 +1,89 @@ -//! Peer discovery for Dash network. -//! -//! Peer discovery is seeded from two sources, in priority order: -//! -//! 1. A hardcoded masternode IP list for the network, embedded at compile time -//! from `dash-spv/seeds/.txt`. This file is regenerated weekly by -//! CI from a live Dash Core node (see `masternode-seeds-fetcher`). -//! 2. DNS seed queries as a backup. DNS resolution failures are logged but are -//! not fatal — as long as the embedded list yields at least one peer, the -//! client can bootstrap. -//! -//! Results from both sources are merged and deduplicated. - -use dashcore::Network; use std::net::SocketAddr; +use std::path::{Path, PathBuf}; -/// DNS discovery for finding initial peers. -/// -/// Despite the name (kept for backwards compatibility), this type also returns -/// hardcoded masternode seeds embedded at compile time; DNS is used as a -/// fallback. -#[derive(Default)] -pub struct DnsDiscovery {} +use dashcore::Network; +use rand::seq::SliceRandom; + +use crate::network::peer::DisconnectedPeer; +use crate::storage::{PeerStorage, PersistentPeerStorage, PersistentStorage}; +use crate::ClientConfig; + +pub struct PeerDiscoverer { + network: Network, + // Empty means "discover": the persisted peer store first, then the seeds. + fixed: Vec, + // When true, hand out only `fixed`: never touch the peer store, seeds, or DNS. + // With no configured peers this yields an empty set, so no outbound connections + // are made — honoring `ClientConfig::restrict_to_configured_peers`. + restrict_to_configured_peers: bool, + storage_path: PathBuf, + /// Discovered addresses, resolved once and then kept. + /// + /// Deliberately not consumed as it is handed out: the reconnector comes back here + /// every time the peer set drops, and a pool that drained itself would leave a client + /// with one known peer unable to reconnect after its second disconnect. + discovered: Option>, +} -impl DnsDiscovery { - /// Create a new DNS discovery instance - pub fn new() -> Self { - Self {} +impl PeerDiscoverer { + pub fn new(config: &ClientConfig) -> PeerDiscoverer { + PeerDiscoverer { + network: config.network, + fixed: config.peers.clone(), + restrict_to_configured_peers: config.restrict_to_configured_peers, + storage_path: config.storage_path.clone(), + discovered: None, + } } - /// Discover peers for the given network. - /// - /// Returns the union of the embedded hardcoded masternode seeds and any - /// addresses resolved via DNS. DNS resolution failures are logged at warn - /// level but do not cause this function to fail — the embedded list acts - /// as the primary source and DNS is a best-effort backup. - pub async fn discover_peers(&self, network: Network) -> Vec { - let seeds = network.dns_seeds(); - let port = network.default_p2p_port(); - let mut addresses = dash_network_seeds::addresses(network); + /// Up to `count` addresses to try, sampled at random from whatever source applies. + pub async fn get(&mut self, count: usize) -> Vec { + let pool = if !self.fixed.is_empty() { + &self.fixed + } else if self.restrict_to_configured_peers { + // Restricted to configured peers, of which there are none: hand out + // nothing rather than falling back to the peer store, seeds, or DNS. + return Vec::new(); + } else { + if self.discovered.is_none() { + let found = Self::discover(self.network, &self.storage_path).await; + self.discovered = Some(found); + } + self.discovered.as_ref().expect("just set") + }; + + pool.choose_multiple(&mut rand::thread_rng(), count) + .map(|addr| DisconnectedPeer::new(*addr, self.network)) + .collect() + } - let embedded_count = addresses.len(); - tracing::info!("Loaded {} hardcoded masternode seed(s) for {:?}", embedded_count, network); + /// Addresses to try, best source first: peers we have actually talked to before, then + /// the compiled-in seeds, then DNS. + async fn discover(network: Network, storage_path: &Path) -> Vec { + let mut addresses = Vec::new(); + + // Peers persisted by an earlier run. On regtest/devnet this is the ONLY source — + // there are no DNS seeds to fall back to — so a client configured without explicit + // peers depends on it entirely. + match PersistentPeerStorage::open(storage_path.to_path_buf()).await { + Ok(store) => match store.load_peers().await { + Ok(peers) => { + let known: Vec = + peers.iter().filter_map(|p| p.socket_addr().ok()).collect(); + if !known.is_empty() { + tracing::info!("Peer store: {} known address(es)", known.len()); + } + addresses.extend(known); + } + Err(e) => tracing::warn!("Failed to load the peer store: {e}"), + }, + Err(e) => tracing::warn!("Failed to open the peer store: {e}"), + } - for seed in seeds { - tracing::debug!("Querying DNS seed: {}", seed); + addresses.extend(dash_network_seeds::addresses(network)); + let port = network.default_p2p_port(); + for seed in network.dns_seeds() { match tokio::net::lookup_host((*seed, port)).await { Ok(iter) => { let resolved: Vec = iter.collect(); @@ -52,7 +91,6 @@ impl DnsDiscovery { addresses.extend(resolved); } Err(e) => { - // DNS is a best-effort backup; do not propagate the error. tracing::warn!("Failed to resolve DNS seed {} (backup source): {}", seed, e); } } @@ -61,68 +99,6 @@ impl DnsDiscovery { addresses.sort(); addresses.dedup(); - tracing::info!( - "Discovered {} unique peer addresses for {:?} ({} from embedded seeds + DNS)", - addresses.len(), - network, - embedded_count - ); addresses } - - /// Discover peers with a limit on the number returned - pub async fn discover_peers_limited(&self, network: Network, limit: usize) -> Vec { - let mut peers = self.discover_peers(network).await; - peers.truncate(limit); - peers - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - #[ignore] // Requires network access - async fn test_dns_discovery_mainnet() { - let discovery = DnsDiscovery::new(); - let peers = discovery.discover_peers(Network::Mainnet).await; - - // Print discovered peers for debugging - println!("Discovered {} mainnet peers:", peers.len()); - for peer in &peers { - println!(" {}", peer); - } - - // All peers should use the correct port - for peer in &peers { - assert_eq!(peer.port(), Network::Mainnet.default_p2p_port()); - } - } - - #[tokio::test] - async fn test_dns_discovery_testnet_returns_embedded_when_dns_fails() { - // This test does not require network access: even if DNS resolution - // fails, the embedded seed file must yield peers. - let discovery = DnsDiscovery::new(); - let peers = discovery.discover_peers(Network::Testnet).await; - - assert!( - peers.len() >= 29, - "expected at least the 29 embedded testnet HP-MN seeds, got {}", - peers.len() - ); - for peer in &peers { - assert_eq!(peer.port(), Network::Testnet.default_p2p_port()); - } - } - - #[tokio::test] - async fn test_dns_discovery_regtest() { - let discovery = DnsDiscovery::new(); - let peers = discovery.discover_peers(Network::Regtest).await; - - // Should return empty for regtest (no DNS seeds and no embedded list) - assert!(peers.is_empty()); - } } diff --git a/dash-spv/src/network/event.rs b/dash-spv/src/network/event.rs deleted file mode 100644 index 397ebd862..000000000 --- a/dash-spv/src/network/event.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Network event system for peer connection state changes. -//! -//! This module provides events for network layer changes that sync managers -//! need to react to, such as peer connections and disconnections. - -use dashcore::prelude::CoreBlockHeight; -use std::fmt; -use std::net::SocketAddr; - -/// Events emitted by the network layer. -/// -/// These events inform sync managers about network state changes, -/// allowing them to wait for connections before sending requests. -#[derive(Debug, Clone)] -pub enum NetworkEvent { - /// A peer has connected. - PeerConnected { - /// Socket address of the connected peer. - address: SocketAddr, - }, - - /// A peer has disconnected. - PeerDisconnected { - /// Socket address of the disconnected peer. - address: SocketAddr, - }, - - /// Summary of connected peers (emitted after connect/disconnect). - /// - /// This event provides the current state of connections after any change. - PeersUpdated { - /// Number of currently connected peers. - connected_count: usize, - /// Addresses of all connected peers. - addresses: Vec, - /// Best height of connected peers. - best_height: Option, - }, -} - -impl fmt::Display for NetworkEvent { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - NetworkEvent::PeerConnected { - address, - } => write!(f, "PeerConnected({})", address), - NetworkEvent::PeerDisconnected { - address, - } => write!(f, "PeerDisconnected({})", address), - NetworkEvent::PeersUpdated { - connected_count, - addresses: _, - best_height, - } => write!( - f, - "PeersUpdated(connected={}, best_height={})", - connected_count, - best_height.unwrap_or(0) - ), - } - } -} diff --git a/dash-spv/src/network/handshake.rs b/dash-spv/src/network/handshake.rs deleted file mode 100644 index 3fc71bd5c..000000000 --- a/dash-spv/src/network/handshake.rs +++ /dev/null @@ -1,320 +0,0 @@ -//! Network handshake management. - -use std::net::SocketAddr; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -use dashcore::network::constants; -use dashcore::network::constants::{ServiceFlags, NODE_HEADERS_COMPRESSED}; -use dashcore::network::message::NetworkMessage; -use dashcore::network::message_network::VersionMessage; -use dashcore::Network; -// Hash trait not needed in current implementation - -use crate::error::{NetworkError, NetworkResult}; -use crate::network::peer::Peer; -use crate::network::Message; - -/// Handshake state. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum HandshakeState { - /// Initial state. - Init, - /// Version message sent. - VersionSent, - /// Version received and verack sent. - VersionReceivedVerackSent, - /// Verack received. - VerackReceived, - /// Handshake complete. - Complete, -} - -/// Manages the network handshake process. -pub struct HandshakeManager { - _network: Network, - state: HandshakeState, - our_version: u32, - peer_version: Option, - peer_services: Option, - version_received: bool, - verack_received: bool, - version_sent: bool, - user_agent: Option, -} - -impl HandshakeManager { - /// Create a new handshake manager. - pub fn new(network: Network, user_agent: Option) -> Self { - Self { - _network: network, - state: HandshakeState::Init, - our_version: constants::PROTOCOL_VERSION, - peer_version: None, - peer_services: None, - version_received: false, - verack_received: false, - version_sent: false, - user_agent, - } - } - - /// Perform the handshake with a peer. - pub async fn perform_handshake(&mut self, connection: &mut Peer) -> NetworkResult<()> { - use tokio::time::{timeout, Duration}; - - // Send version message - self.send_version(connection).await?; - self.version_sent = true; - self.state = HandshakeState::VersionSent; - tracing::info!("Handshake initiated - version message sent to peer"); - - // Define timeout for the entire handshake process - const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); - const MESSAGE_POLL_INTERVAL: Duration = Duration::from_millis(100); - - let start_time = tokio::time::Instant::now(); - - // Wait for responses with timeout - loop { - // Check if we've exceeded the overall handshake timeout - if start_time.elapsed() > HANDSHAKE_TIMEOUT { - tracing::error!( - "Handshake timeout after {}s - version_received={}, verack_received={}", - HANDSHAKE_TIMEOUT.as_secs(), - self.version_received, - self.verack_received - ); - return Err(NetworkError::Timeout); - } - - // Try to receive a message with a short timeout - match timeout(MESSAGE_POLL_INTERVAL, connection.receive_message()).await { - Ok(Ok(Some(message))) => { - tracing::debug!("Received message during handshake: {:?}", message.cmd()); - match self.handle_handshake_message(connection, &message).await? { - Some(HandshakeState::Complete) => { - self.state = HandshakeState::Complete; - break; - } - _ => { - // Continue immediately to check for more messages in the buffer - // Don't add any delays here as multiple messages may be waiting - continue; - } - } - } - Ok(Ok(None)) => { - // No message available, continue immediately - // The read timeout already provides the necessary delay - continue; - } - Ok(Err(e)) => { - tracing::error!("Error receiving message during handshake: {}", e); - return Err(e); - } - Err(_) => { - // Timeout on receive_message, continue to check overall timeout - continue; - } - } - } - - tracing::info!( - "Handshake completed successfully - version_received={}, verack_received={}", - self.version_received, - self.verack_received - ); - Ok(()) - } - - /// Reset the handshake state. - pub fn reset(&mut self) { - self.state = HandshakeState::Init; - self.peer_version = None; - self.version_received = false; - self.verack_received = false; - self.version_sent = false; - } - - /// Handle a handshake message. - async fn handle_handshake_message( - &mut self, - connection: &mut Peer, - message: &Message, - ) -> NetworkResult> { - match message.inner() { - NetworkMessage::Version(version_msg) => { - tracing::debug!( - "Peer {} sent version message: {:?}", - message.peer_address(), - version_msg - ); - self.peer_version = Some(version_msg.version); - self.peer_services = Some(version_msg.services); - self.version_received = true; - - // Update connection's peer information - connection.update_peer_info(version_msg); - - // If we haven't sent our version yet (peer initiated), send it now - if !self.version_sent { - tracing::debug!( - "Peer {} initiated handshake, sending our version", - message.peer_address() - ); - self.send_version(connection).await?; - self.version_sent = true; - } - - // Send SendAddrV2 first to signal support (must be before verack!) - tracing::debug!("Sending sendaddrv2 to signal AddrV2 support"); - connection.send_message(NetworkMessage::SendAddrV2).await?; - - // Then send verack - tracing::debug!("Sending verack in response to version"); - connection.send_message(NetworkMessage::Verack).await?; - tracing::debug!( - "Sent verack, version_received={}, verack_received={}", - self.version_received, - self.verack_received - ); - - // Update state - self.state = HandshakeState::VersionReceivedVerackSent; - - // Check if handshake is complete (both version and verack received) - if self.version_received && self.verack_received { - tracing::info!("Handshake complete - both version and verack exchanged!"); - - // Negotiate headers2 support - self.negotiate_headers2(connection).await?; - - return Ok(Some(HandshakeState::Complete)); - } - - Ok(None) - } - NetworkMessage::Verack => { - tracing::debug!("Received verack message, current state: {:?}", self.state); - self.verack_received = true; - - // Update state - if self.state == HandshakeState::VersionSent { - self.state = HandshakeState::VerackReceived; - } - - // Check if handshake is complete (both version and verack received) - if self.version_received && self.verack_received { - tracing::info!("Handshake complete - both version and verack exchanged!"); - - // Negotiate headers2 support - self.negotiate_headers2(connection).await?; - - return Ok(Some(HandshakeState::Complete)); - } else { - tracing::debug!( - "Verack received but handshake not complete: version_received={}, verack_received={}", - self.version_received, self.verack_received - ); - } - Ok(None) - } - NetworkMessage::Ping(nonce) => { - // Respond to ping during handshake - tracing::debug!("Responding to ping during handshake: {}", nonce); - connection.send_message(NetworkMessage::Pong(*nonce)).await?; - Ok(None) - } - NetworkMessage::SendAddrV2 => { - // Peer supports AddrV2 - tracing::debug!("Peer signaled AddrV2 support"); - Ok(None) - } - _ => { - // Ignore other messages during handshake - tracing::debug!("Ignoring message during handshake: {:?}", message); - Ok(None) - } - } - } - - /// Send version message. - async fn send_version(&mut self, connection: &mut Peer) -> NetworkResult<()> { - let version_message = self.build_version_message(connection.address())?; - connection.send_message(NetworkMessage::Version(version_message)).await?; - tracing::debug!("Sent version message"); - Ok(()) - } - - /// Build version message. - fn build_version_message(&self, address: SocketAddr) -> NetworkResult { - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or(Duration::from_secs(0)) - .as_secs() as i64; - - // Advertise headers2 support (NODE_HEADERS_COMPRESSED) - let services = ServiceFlags::NONE | NODE_HEADERS_COMPRESSED; - - // Parse the local address safely - let local_addr = "127.0.0.1:0" - .parse() - .map_err(|_| NetworkError::AddressParse("Failed to parse local address".to_string()))?; - - // Determine user agent: prefer configured value, else default to crate/version. - let default_agent = format!("/rust-dash-spv:{}/", env!("CARGO_PKG_VERSION")); - let mut ua = self.user_agent.clone().unwrap_or(default_agent); - // Normalize: ensure it starts and ends with '/'; trim if excessively long. - if !ua.starts_with('/') { - ua.insert(0, '/'); - } - if !ua.ends_with('/') { - ua.push('/'); - } - // Keep within a reasonable bound (match peer validation bound of 256) - if ua.len() > 256 { - ua.truncate(256); - } - - Ok(VersionMessage { - version: self.our_version, - services, - timestamp, - receiver: dashcore::network::address::Address::new(&address, ServiceFlags::NETWORK), - sender: dashcore::network::address::Address::new(&local_addr, services), - nonce: rand::random(), - user_agent: ua, - start_height: 0, // SPV client starts at 0 - relay: false, // relay enabled on demand via filterload/filterclear - mn_auth_challenge: [0; 32], // Not a masternode - masternode_connection: false, // Not connecting to masternode - }) - } - - /// Get current handshake state. - pub fn state(&self) -> &HandshakeState { - &self.state - } - - /// Get peer version if available. - pub fn peer_version(&self) -> Option { - self.peer_version - } - - /// Check if peer supports headers2 compression. - pub fn peer_supports_headers2(&self) -> bool { - self.peer_services.map(|services| services.has(NODE_HEADERS_COMPRESSED)).unwrap_or(false) - } - - /// Negotiate headers2 support with the peer after handshake completion. - async fn negotiate_headers2(&self, connection: &mut Peer) -> NetworkResult<()> { - if self.peer_supports_headers2() { - tracing::info!("Peer supports headers2 - sending SendHeaders2"); - connection.send_message(NetworkMessage::SendHeaders2).await?; - } else { - tracing::info!("Peer does not support headers2 - sending SendHeaders"); - connection.send_message(NetworkMessage::SendHeaders).await?; - } - Ok(()) - } -} diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 680eedf4d..c77e612eb 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1,1558 +1,1200 @@ -//! Peer network manager for SPV client - -use std::collections::{HashMap, HashSet}; -use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; -use std::path::PathBuf; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{broadcast, Mutex, RwLock}; -use tokio::task::JoinSet; -use tokio::time; - -use crate::client::ClientConfig; -use crate::error::{NetworkError, NetworkResult, SpvError as Error}; -use crate::network::addrv2::AddrV2Handler; -use crate::network::constants::*; -use crate::network::discovery::DnsDiscovery; -use crate::network::pool::PeerPool; -use crate::network::reputation::{ChangeReason, PeerReputationManager, ReputationAware}; -use crate::network::{ - HandshakeManager, Message, MessageDispatcher, MessageType, NetworkEvent, NetworkManager, - NetworkRequest, Peer, RequestSender, +use std::{ + collections::{HashMap, HashSet, VecDeque}, + net::SocketAddr, + sync::{ + atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, }; -use crate::storage::{PeerStorage, PersistentPeerStorage, PersistentStorage}; -use async_trait::async_trait; -use dashcore::network::address::{AddrV2, AddrV2Message}; -use dashcore::network::constants::ServiceFlags; + use dashcore::network::message::NetworkMessage; -use dashcore::network::message_headers2::CompressionState; -use dashcore::Network; -use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; -use tokio::time::Instant; +use dashcore::network::message_blockdata::Inventory; +use futures::future::join_all; +use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; +use tokio::sync::{broadcast, Mutex, Notify}; +use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; -const DEFAULT_NETWORK_EVENT_CAPACITY: usize = 10000; +const HIGH_DEMAND_QUEUE: usize = 40; +const ADD_PEERS_BATCH: usize = 4; +const PROBE_BATCH: usize = 256; +/// Safety ceiling on how many peers backpressure may auto-connect. `max_peers` +/// (config) is the base/initial set; when the send queue stays backed up the +/// peers can't serve fast enough, so we recruit more — up to this cap — to raise +/// aggregate serving capacity. +/// +/// Kept near the empirical sweet spot: on testnet, throughput improves up to +/// ~16 peers, is flat-to-worse by 24-32, and OUTRIGHT STALLS around 64 — dozens +/// of slow peers each pin their measured in-flight on batches that never finish, +/// so there are no completions to wake the router. More peers past the knee only +/// adds congestion, not download rate (the peers, not our link, are the limit). +const MAX_CONNECTED_PEERS: usize = 16; + +/// How often the reconnector checks whether the peer set needs topping up. +const RECONNECT_CHECK: Duration = Duration::from_secs(2); + +/// How long the router waits on a full-capacity stall before checking whether the +/// "in-flight" requests holding that capacity are actually dead. +const STALL_CHECK: Duration = Duration::from_secs(5); + +/// A request unanswered for this long has its slot reclaimed: the owning pipeline +/// has long since timed it out and re-queued it elsewhere, so the slot holds +/// nothing. +/// +/// Generous, because slow is not dead. Blocks are paced through the same budget, +/// and a 2MB block over a weak link legitimately takes a while — the point is to +/// free capacity, not to punish peers (whether the peer is dropped is decided +/// separately, on whether it has EVER answered). +const STALE_REQUEST: Duration = Duration::from_secs(90); +use crate::{ + network::{ + discovery::PeerDiscoverer, + peer::{ConnectedPeer, DisconnectedPeer, PeerEvent}, + }, + ClientConfig, +}; -/// Peer network manager -pub struct PeerNetworkManager { - /// Peer pool - pool: Arc, - max_peers: usize, - /// DNS discovery - discovery: Arc, - /// AddrV2 handler - addrv2_handler: Arc, - /// Peer persistence - peer_store: Arc, - /// Peer reputation manager - reputation_manager: Arc, - /// Network type - network: Network, - /// Shutdown token - shutdown_token: CancellationToken, - /// Background tasks - tasks: Arc>>, - /// Initial peer addresses - initial_peers: Vec, - /// Data directory for storage - data_dir: PathBuf, - /// Optional user agent to advertise - user_agent: Option, - /// Exclusive mode: restrict to configured peers only (no DNS or peer store) - exclusive_mode: bool, - /// Service flags connected peers must advertise. NONE disables capability churn. - required_services: ServiceFlags, - /// Addresses evicted for lacking required services. Excluded from top-up candidates. - /// TODO: remove once peer session outcomes track why sessions ended and drive reconnect policy. - capability_rejected: Arc>>, - /// Cached count of currently connected peers for fast, non-blocking queries - connected_peer_count: Arc, - /// Disable headers2 after decompression failure - headers2_disabled: Arc>>, - /// Dispatcher for unbounded and message-type filtered message distribution. - message_dispatcher: Arc>, - /// Request queue sender, cloneable handle for sending requests to the network manager. - request_tx: UnboundedSender, - /// Request queue receiver (consumed by send loop). - request_rx: Arc>>>, - /// Round-robin counter for distributing requests across peers. - round_robin_counter: Arc, - /// Network event bus for notifying about network/peer related changes. - network_event_sender: broadcast::Sender, +/// An inbound message on its way to the managers that subscribed to its type. +/// +/// Shared rather than cloned: a `block` or `cfilter` carries its whole payload, and the +/// pump would otherwise deep-copy it for every subscriber. See `spawn_pump`'s fan-out. +type Inbound = (SocketAddr, Arc); +type Subscribers = Arc>>>>; + +/// The kinds of peer message a sync manager can subscribe to. Replaces the +/// stringly-typed command names: managers declare interest with these variants +/// and the pump routes incoming messages by mapping `cmd()` back to one. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum MessageType { + Headers, + Inv, + CfHeaders, + CFilter, + Block, + MnListDiff, + QrInfo, + Tx, + IsDLock, + ChainLock, } -const CAPABILITY_REJECTED_TTL: Duration = Duration::from_secs(30 * 60); - -fn required_services_from_config(config: &ClientConfig, exclusive_mode: bool) -> ServiceFlags { - if exclusive_mode { - return ServiceFlags::NONE; +impl MessageType { + /// The wire command string this type corresponds to. + pub fn cmd(self) -> &'static str { + match self { + MessageType::Headers => "headers", + MessageType::Inv => "inv", + MessageType::CfHeaders => "cfheaders", + MessageType::CFilter => "cfilter", + MessageType::Block => "block", + MessageType::MnListDiff => "mnlistdiff", + MessageType::QrInfo => "qrinfo", + MessageType::Tx => "tx", + MessageType::IsDLock => "isdlock", + MessageType::ChainLock => "clsig", + } } - let mut flags = ServiceFlags::NONE; - if config.enable_filters { - flags |= ServiceFlags::COMPACT_FILTERS; + + /// Map an incoming message's command back to a subscribed type, if any. + pub fn from_cmd(cmd: &str) -> Option { + Some(match cmd { + "headers" => MessageType::Headers, + "inv" => MessageType::Inv, + "cfheaders" => MessageType::CfHeaders, + "cfilter" => MessageType::CFilter, + "block" => MessageType::Block, + "mnlistdiff" => MessageType::MnListDiff, + "qrinfo" => MessageType::QrInfo, + "tx" => MessageType::Tx, + "isdlock" => MessageType::IsDLock, + "clsig" => MessageType::ChainLock, + _ => return None, + }) } - flags } -impl PeerNetworkManager { - /// Create a new peer network manager - pub async fn new(config: &ClientConfig) -> Result { - let discovery = DnsDiscovery::new(); - let data_dir = config.storage_path.clone(); +#[derive(Clone, Debug)] +pub enum NetworkEvent { + /// The connected peer set changed. + PeersUpdated { + /// How many peers are currently connected. + connected_count: u32, + /// Best tip height advertised across those peers. + best_height: u32, + }, + PeerConnected(SocketAddr), + PeerDisconnected(SocketAddr), + /// A pipeline request just left the router for a peer (it is now on the + /// wire). Pipelines use this to start the request's response timeout from the + /// actual send, not from when it was queued (which fires spuriously when the + /// router is backed up). + RequestOnFlight(RequestKey), +} + +/// Identifies a pipeline request that has been sent, so the owning pipeline can +/// match it to its in-flight entry and start the timeout. One variant per +/// router-paced request type. +#[derive(Clone, Debug)] +pub enum RequestKey { + /// `getheaders` — keyed by the locator's first hash (the segment tip). + Headers(dashcore::BlockHash), + /// `getcfheaders` — keyed by the stop hash. + CfHeaders(dashcore::BlockHash), + /// `getcfilters` — keyed by the start height. + CFilters(u32), + /// `getmnlistdiff` — keyed by the target block hash. + MnListDiff(dashcore::BlockHash), + /// Block `getdata` — keyed by the requested block hash. + Block(dashcore::BlockHash), +} - let peer_store = PersistentPeerStorage::open(data_dir.clone()).await?; +pub struct PeerNetworkManager { + connected_peers: Arc>>, + other_peers: Arc>>, + discoverer: Arc>, + msg_queue: Arc, + inbound_tx: UnboundedSender, + subscribers: Subscribers, + events_tx: broadcast::Sender, + /// Best tip advertised by the peers, learned in `start`. Shared with the + /// reconnector so its `PeersUpdated` carries the real height. + best_tip: Arc, + max_peers: usize, + /// Total bytes read from all peers. Held so `start` can hand it to the peers it connects. + bytes: Arc, + // Cancelled by `stop()` to tear down the router, pump and every peer reader. + shutdown: CancellationToken, +} - let reputation_manager = Arc::new(PeerReputationManager::new()); +/// What kind of work a queued message is, for scheduling. A single FIFO let one +/// pipeline monopolise the router: a filters phase queues batches by the hundred, +/// so a `getcfheaders` (or a small control message) landing behind them waited for +/// the whole backlog — the phases then advance one after another instead of +/// together. The router drains `Other` first and round-robins the three bulk +/// classes, so every pipeline keeps making progress. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum MsgClass { + /// Small/control messages (mempool, tx, ping, chainlock/islock `getdata`…): + /// always first, they are few and something is usually blocked on them. + Other, + Headers, + CfHeaders, + CFilters, + /// Block `getdata` — the wallet's matched blocks. + Blocks, +} - if let Err(e) = reputation_manager.load_from_storage(&peer_store).await { - tracing::warn!("Failed to load peer reputation data: {}", e); +fn classify(msg: &NetworkMessage) -> MsgClass { + match msg { + NetworkMessage::GetHeaders(_) | NetworkMessage::GetHeaders2(_) => MsgClass::Headers, + NetworkMessage::GetCFHeaders(_) => MsgClass::CfHeaders, + NetworkMessage::GetCFilters(_) => MsgClass::CFilters, + NetworkMessage::GetData(inv) + if !inv.is_empty() && inv.iter().all(|i| matches!(i, Inventory::Block(_))) => + { + MsgClass::Blocks } + _ => MsgClass::Other, + } +} - // Determine exclusive mode: either explicitly requested or peers were provided - let exclusive_mode = config.restrict_to_configured_peers || !config.peers.is_empty(); - let required_services = required_services_from_config(config, exclusive_mode); +/// One queue per class, each behind its own lock: a pipeline enqueuing a burst +/// only contends with itself, never with the others. +struct MsgQueue { + other: Mutex>, + headers: Mutex>, + cfheaders: Mutex>, + cfilters: Mutex>, + blocks: Mutex>, + + /// Round-robin cursor over the three bulk classes. + turn: AtomicUsize, + len: AtomicUsize, + notify: Notify, +} - // Create request queue for outgoing messages - let (request_tx, request_rx) = unbounded_channel(); +struct State {} +impl PeerNetworkManager { + pub async fn new(config: &ClientConfig) -> Self { + let discoverer = Arc::new(Mutex::new(PeerDiscoverer::new(config))); let max_peers = config.max_peers.max(1) as usize; - Ok(Self { - pool: Arc::new(PeerPool::new(max_peers)), - max_peers, - discovery: Arc::new(discovery), - addrv2_handler: Arc::new(AddrV2Handler::new()), - peer_store: Arc::new(peer_store), - reputation_manager, - network: config.network, - shutdown_token: CancellationToken::new(), - tasks: Arc::new(Mutex::new(JoinSet::new())), - initial_peers: config.peers.clone(), - data_dir, - user_agent: config.user_agent.clone(), - exclusive_mode, - required_services, - capability_rejected: Arc::new(RwLock::new(HashMap::new())), - connected_peer_count: Arc::new(AtomicUsize::new(0)), - headers2_disabled: Arc::new(Mutex::new(HashSet::new())), - message_dispatcher: Arc::new(Mutex::new(MessageDispatcher::default())), - request_tx, - request_rx: Arc::new(Mutex::new(Some(request_rx))), - round_robin_counter: Arc::new(AtomicUsize::new(0)), - network_event_sender: broadcast::Sender::new(DEFAULT_NETWORK_EVENT_CAPACITY), - }) - } + let connected_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); + let other_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); + let msg_queue = Arc::new(MsgQueue::new()); + + let (inbound_tx, inbound_rx) = mpsc::unbounded_channel(); + let subscribers: Subscribers = Arc::new(Mutex::new(HashMap::new())); + // Sized generously: `RequestOnFlight` is emitted per sent request, so the + // network-event bus is higher-volume than the peer-state events alone. + let (events_tx, _) = broadcast::channel(4096); + let shutdown = CancellationToken::new(); + // Total bytes read from all peers (download-only) and the global in-flight + // budget. The budget is NOT a fixed number: it starts at a tiny bootstrap + // just large enough to begin measuring, then `spawn_bandwidth_controller` + // sizes it from the measured download capacity (Little's Law). + let bytes = Arc::new(AtomicU64::new(0)); + let global_cap = Arc::new(AtomicUsize::new(max_peers.saturating_mul(4).max(8))); + let best_tip = Arc::new(AtomicU32::new(0)); + + // Detached like the bandwidth controller and reconnector below: torn down + // via the shutdown token, not by holding their handles. + spawn_pump( + inbound_rx, + subscribers.clone(), + connected_peers.clone(), + events_tx.clone(), + msg_queue.clone(), + shutdown.clone(), + ); - /// Creates and returns a receiver that yields only messages of the matching the provided message types. - pub async fn message_receiver( - &mut self, - message_types: &[MessageType], - ) -> UnboundedReceiver { - self.message_dispatcher.lock().await.message_receiver(message_types) - } + spawn_router( + msg_queue.clone(), + connected_peers.clone(), + other_peers.clone(), + inbound_tx.clone(), + shutdown.clone(), + bytes.clone(), + global_cap.clone(), + events_tx.clone(), + ); - /// Get a RequestSender for queueing outgoing network requests. - pub fn request_sender(&self) -> RequestSender { - RequestSender::new(self.request_tx.clone()) - } + spawn_bandwidth_controller( + bytes.clone(), + global_cap.clone(), + connected_peers.clone(), + shutdown.clone(), + ); - /// Get the network event bus for sharing with other components. - pub fn network_event_sender(&self) -> &broadcast::Sender { - &self.network_event_sender + spawn_reconnector( + discoverer.clone(), + connected_peers.clone(), + other_peers.clone(), + inbound_tx.clone(), + shutdown.clone(), + bytes.clone(), + events_tx.clone(), + best_tip.clone(), + max_peers, + ); + + PeerNetworkManager { + connected_peers, + other_peers, + discoverer, + msg_queue, + inbound_tx, + subscribers, + events_tx, + best_tip, + max_peers, + bytes, + shutdown, + } } - /// Start the network manager - pub async fn start(&self) -> Result<(), Error> { - tracing::info!("Starting peer network manager for {:?}", self.network); + /// Connect to peers and announce them. + /// + /// Split out of `new` on purpose: connecting there meant the one-shot `PeersUpdated` + /// (and every `PeerConnected`) fired before any sync manager had subscribed, so those + /// events were simply lost. Managers that track the peer set — the mempool, which must + /// send `filterload` to enable transaction relay — ended up with an empty set and never + /// activated. Build the manager, let the coordinator spawn and subscribe its managers, + /// then call this. + pub async fn start(&self) { + let best_tip = probe_and_select( + &mut *self.discoverer.lock().await, + &self.connected_peers, + &self.other_peers, + self.max_peers, + &self.inbound_tx, + &self.shutdown, + &self.bytes, + ) + .await; - let mut peer_addresses: Vec = self - .initial_peers - .iter() - .map(|addr| AddrV2Message::new(*addr, ServiceFlags::NETWORK)) - .collect(); + self.best_tip.store(best_tip, Ordering::Relaxed); - if self.exclusive_mode { - tracing::info!( - "Exclusive peer mode: connecting ONLY to {} specified peer(s)", - self.initial_peers.len() - ); - } else { - // Load saved peers from disk - let saved_peers = self.peer_store.load_peers().await.unwrap_or_else(|e| { - tracing::warn!("Failed to load peers: {}", e); - Vec::new() - }); - peer_addresses.extend(saved_peers); - - // If we still have no peers, immediately discover via DNS - if peer_addresses.is_empty() { - tracing::info!( - "No peers configured, performing immediate DNS discovery for {:?}", - self.network - ); - let dns_peers = self.discovery.discover_peers(self.network).await; - let dns_peers_found = dns_peers.len(); - peer_addresses.extend( - dns_peers - .into_iter() - .take(self.max_peers) - .map(|addr| AddrV2Message::new(addr, ServiceFlags::NETWORK)), - ); - tracing::info!( - "DNS discovery found {} peers, using {} for startup", - dns_peers_found, - peer_addresses.len() - ); - } else { - tracing::info!( - "Starting with {} peers from disk (DNS discovery will be used later if needed)", - peer_addresses.len() - ); - } + // Announce each peer individually before the summary. Managers that track the peer + // set (the mempool, which must send `filterload` to turn on transaction relay) build + // it from these; `PeersUpdated` alone carries no addresses. + let addrs: Vec = + self.connected_peers.lock().await.iter().map(|(peer, _)| peer.addr()).collect(); + let connected_count = addrs.len() as u32; + for addr in addrs { + let _ = self.events_tx.send(NetworkEvent::PeerConnected(addr)); } - self.addrv2_handler.handle_addrv2(peer_addresses.clone()).await; - - // Start maintenance loop - self.start_maintenance_loop().await; + let _ = self.events_tx.send(NetworkEvent::PeersUpdated { + connected_count, + best_height: best_tip, + }); + } - // Start request processing task for managers to queue outgoing messages - self.start_request_processor().await; + /// Tear down the network layer: stop the router and pump, and cancel every + /// peer reader so no more messages arrive. Called on client shutdown. + pub fn stop(&self) { + tracing::info!(target: "dash_spv::network", "network manager stopping: cancelling tasks and peers"); + self.shutdown.cancel(); + } - Ok(()) + pub async fn send(&self, msg: NetworkMessage) { + self.msg_queue.push(msg).await; } - /// Connect to a specific peer - async fn connect_to_peer(&self, addr: SocketAddr) { - // Check reputation first - if !self.reputation_manager.should_connect_to_peer(&addr).await { - tracing::warn!("Not connecting to {} due to bad reputation", addr); - return; + /// Send a message to one specific peer, bypassing the router's round-robin. + /// + /// `send` hands a message to whichever peer has capacity, which is right for a request + /// any peer can answer. It is wrong for a message that sets state ON the remote node: + /// `filterload`/`filterclear` (and the `mempool` that follows) turn transaction relay + /// on for THAT peer, so routing them to "whoever is free" leaves the intended peer + /// silent — and, with several peers, can enable relay on the same one twice. + /// + /// Returns false if the peer is not connected (or the write failed). + pub async fn send_to(&self, addr: SocketAddr, msg: NetworkMessage) -> bool { + let peers = self.connected_peers.lock().await; + let Some((peer, _)) = peers.iter().find(|(p, _)| p.addr() == addr) else { + return false; + }; + + match peer.send(&msg).await { + Ok(()) => true, + Err(e) => { + tracing::warn!(target: "dash_spv::network", "send to {addr} failed: {e}"); + false + } } + } - // Check if already connected or connecting - if self.pool.is_connected(&addr).await || self.pool.is_connecting(&addr).await { + /// Note that `n` streaming requests served by `peer` have fully completed + /// (e.g. a `getcfilters` batch whose last `cfilter` just arrived), freeing + /// that peer's in-flight units and waking the router. Single-response + /// requests are freed in the peer's own reader instead. + pub async fn request_completed(&self, peer: SocketAddr, n: usize) { + if n == 0 { return; } - - // Mark as connecting - if !self.pool.mark_connecting(addr).await { - return; // Already being connected to + if let Some((p, _)) = + self.connected_peers.lock().await.iter().find(|(p, _)| p.addr() == peer) + { + p.response_completed(n).await; } + self.msg_queue.notify.notify_one(); + } - // Record connection attempt - self.reputation_manager.record_connection_attempt(addr).await; - - let pool = self.pool.clone(); - let network = self.network; - let addrv2_handler = self.addrv2_handler.clone(); - let shutdown_token = self.shutdown_token.clone(); - let reputation_manager = self.reputation_manager.clone(); - let user_agent = self.user_agent.clone(); - let required_services = self.required_services; - let capability_rejected = self.capability_rejected.clone(); - let connected_peer_count = self.connected_peer_count.clone(); - let headers2_disabled = self.headers2_disabled.clone(); - let message_dispatcher = self.message_dispatcher.clone(); - let network_event_sender = self.network_event_sender.clone(); - - // Spawn connection task — use select to avoid blocking on the lock during shutdown - let mut tasks = tokio::select! { - guard = self.tasks.lock() => guard, - _ = self.shutdown_token.cancelled() => { - self.pool.remove_peer(&addr).await; - return; - } - }; - tasks.spawn(async move { - tracing::debug!("Attempting to connect to {}", addr); - - let connect_result = tokio::select! { - result = Peer::connect(addr, CONNECTION_TIMEOUT.as_secs(), network) => result, - _ = shutdown_token.cancelled() => { - tracing::debug!("Connection to {} cancelled by shutdown", addr); - pool.remove_peer(&addr).await; - return; - } - }; - - match connect_result { - Ok(mut peer) => { - // Perform handshake - let mut handshake_manager = HandshakeManager::new(network, user_agent); - match handshake_manager.perform_handshake(&mut peer).await { - Ok(_) => { - if PeerNetworkManager::should_reject_after_handshake( - &pool, - &peer, - required_services, - ) - .await - { - tracing::info!( - "Rejecting peer {} during handshake - missing required services ({}) while a capable peer is connected", - addr, - required_services - ); - PeerNetworkManager::record_capability_rejection_in( - &capability_rejected, - addr, - ) - .await; - pool.remove_peer(&addr).await; - return; - } - tracing::info!("Successfully connected to {}", addr); - - // Request addresses from the peer for discovery - if let Err(e) = peer.send_message(NetworkMessage::GetAddr).await { - tracing::warn!("Failed to send GetAddr to {}: {}", addr, e); - } - - // Record successful connection - reputation_manager.record_successful_connection(addr).await; - - // Add to pool - if let Err(e) = pool.add_peer(addr, peer).await { - tracing::error!("Failed to add peer to pool: {}", e); - return; - } - - // Increment connected peer counter on successful add - connected_peer_count.fetch_add(1, Ordering::Relaxed); - - // Emit peer connected event - let count = connected_peer_count.load(Ordering::Relaxed); - let addresses = pool.get_connected_addresses().await; - let best_height = pool.get_best_height().await; - let _ = network_event_sender.send(NetworkEvent::PeerConnected { - address: addr, - }); - let _ = network_event_sender.send(NetworkEvent::PeersUpdated { - connected_count: count, - addresses, - best_height, - }); - - // Add to known addresses - addrv2_handler.add_known_address(addr, ServiceFlags::NETWORK).await; - - // // Start message reader for this peer - Self::start_peer_reader( - addr, - pool.clone(), - addrv2_handler, - shutdown_token, - reputation_manager.clone(), - connected_peer_count.clone(), - headers2_disabled.clone(), - message_dispatcher, - network_event_sender.clone(), - ) - .await; - } - Err(e) => { - tracing::warn!("Handshake failed with {}: {}", addr, e); - // Only clears connecting set. Peer was never added, so no count/event needed. - pool.remove_peer(&addr).await; - // Update reputation for handshake failure - reputation_manager - .update_reputation(addr, ChangeReason::HandshakeFailed) - .await; - // For handshake failures, try again later - tokio::time::sleep(RECONNECT_DELAY).await; - } - } - } - Err(e) => { - tracing::debug!("Failed to connect to {}: {}", addr, e); - // Only clears connecting set. Peer was never added, so no count/event needed. - pool.remove_peer(&addr).await; - // Minor reputation penalty for connection failure - reputation_manager - .update_reputation(addr, ChangeReason::ConnectionFailed) - .await; - } + pub fn broadcast(&self, msg: NetworkMessage) { + let peers = self.connected_peers.clone(); + tokio::spawn(async move { + let guard = peers.lock().await; + for (peer, _) in guard.iter() { + let _ = peer.send(&msg).await; } }); } - /// Decrement the connected count and emit PeerDisconnected / PeersUpdated events. - async fn notify_peer_removed( - pool: &PeerPool, - addr: &SocketAddr, - connected_peer_count: &AtomicUsize, - network_event_sender: &broadcast::Sender, - ) { - let sub_result = - connected_peer_count - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| c.checked_sub(1)); - if sub_result.is_err() { - tracing::warn!("Peer count already zero when removing {}", addr); - } - let count = connected_peer_count.load(Ordering::Relaxed); - let addresses = pool.get_connected_addresses().await; - let best_height = pool.get_best_height().await; - let _ = network_event_sender.send(NetworkEvent::PeerDisconnected { - address: *addr, - }); - let _ = network_event_sender.send(NetworkEvent::PeersUpdated { - connected_count: count, - addresses, - best_height, - }); + /// Inject a message into the local pump as if it arrived from a peer, so + /// managers process it through the same path. Uses the `0.0.0.0:0` sentinel + /// address that managers treat as locally-originated. + pub async fn dispatch_local(&self, msg: NetworkMessage) { + let local: SocketAddr = ([0, 0, 0, 0], 0).into(); + let _ = self.inbound_tx.send(PeerEvent::Message(local, msg)); } - /// Remove a peer from the pool, decrement the connected count, and emit - /// PeerDisconnected / PeersUpdated events. - async fn remove_peer_and_notify( - pool: &PeerPool, - addr: &SocketAddr, - connected_peer_count: &AtomicUsize, - network_event_sender: &broadcast::Sender, - ) { - if pool.remove_peer(addr).await.is_some() { - Self::notify_peer_removed(pool, addr, connected_peer_count, network_event_sender).await; + pub async fn subscribe(&self, kinds: &[MessageType]) -> UnboundedReceiver { + let (tx, rx) = mpsc::unbounded_channel(); + let mut subscribers = self.subscribers.lock().await; + for kind in kinds { + subscribers.entry(*kind).or_default().push(tx.clone()); } + rx } - /// Start reading messages from a peer - #[allow(clippy::too_many_arguments)] // TODO: refactor to reduce arguments - async fn start_peer_reader( - addr: SocketAddr, - pool: Arc, - addrv2_handler: Arc, - shutdown_token: CancellationToken, - reputation_manager: Arc, - connected_peer_count: Arc, - headers2_disabled: Arc>>, - message_dispatcher: Arc>, - network_event_sender: broadcast::Sender, - ) { - tokio::spawn(async move { - tracing::debug!("Starting peer reader loop for {}", addr); - let mut loop_iteration = 0; - let mut headers2_state = CompressionState::default(); - - loop { - loop_iteration += 1; + pub fn tip(&self) -> u32 { + self.best_tip.load(Ordering::Relaxed) + } - // Check shutdown signal first with detailed logging - if shutdown_token.is_cancelled() { - tracing::info!("Breaking peer reader loop for {} - shutdown signal received (iteration {})", addr, loop_iteration); - break; - } + /// How many peers are currently connected. + pub async fn connected_count(&self) -> u32 { + self.connected_peers.lock().await.len() as u32 + } - // Get peer - let peer = match pool.get_peer(&addr).await { - Some(peer) => peer, - None => { - tracing::warn!("Breaking peer reader loop for {} - peer no longer in pool (iteration {})", addr, loop_iteration); - break; - } - }; - - // Read message with minimal lock time - let msg_result = { - // Try to get a read lock first to check if peer is available - let peer_guard = peer.read().await; - if !peer_guard.is_connected() { - tracing::warn!("Breaking peer reader loop for {} - peer no longer connected (iteration {})", addr, loop_iteration); - drop(peer_guard); - break; - } - drop(peer_guard); - - // Now get write lock only for the duration of the read - let mut peer_guard = peer.write().await; - tokio::select! { - message = peer_guard.receive_message() => { - message - }, - _ = tokio::time::sleep(MESSAGE_POLL_INTERVAL) => { - Ok(None) - }, - _ = shutdown_token.cancelled() => { - tracing::info!("Breaking peer reader loop for {} - shutdown signal received while reading (iteration {})", addr, loop_iteration); - break; - } - } - }; - - match msg_result { - Ok(Some(msg)) => { - // Log all received messages at debug level to help troubleshoot - tracing::debug!("Received {:?} from {}", msg.cmd(), addr); - - // Handle some messages directly - match &msg.inner() { - NetworkMessage::SendAddrV2 => { - addrv2_handler.handle_sendaddrv2(addr).await; - continue; // Don't forward to client - } - NetworkMessage::SendHeaders2 => { - // Peer is indicating they will send us compressed headers - tracing::info!( - "Peer {} sent SendHeaders2 - they will send compressed headers", - addr - ); - let mut peer_guard = peer.write().await; - peer_guard.set_peer_sent_sendheaders2(true); - drop(peer_guard); - continue; // Don't forward to client - } - NetworkMessage::AddrV2(addresses) => { - addrv2_handler.handle_addrv2(addresses.clone()).await; - continue; // Don't forward to client - } - NetworkMessage::GetAddr => { - tracing::trace!( - "Received GetAddr from {}, sending known addresses", - addr - ); - // Send our known addresses - let response = addrv2_handler.build_addr_response().await; - let mut peer_guard = peer.write().await; - if let Err(e) = peer_guard.send_message(response).await { - tracing::error!( - "Failed to send addr response to {}: {}", - addr, - e - ); - } - continue; // Don't forward GetAddr to client - } - NetworkMessage::Ping(nonce) => { - // Handle ping directly - let mut peer_guard = peer.write().await; - if let Err(e) = peer_guard.handle_ping(*nonce).await { - tracing::error!("Failed to handle ping from {}: {}", addr, e); - // If we can't send pong, connection is likely broken - if matches!(e, NetworkError::ConnectionFailed(_)) { - tracing::warn!("Breaking peer reader loop for {} - failed to send pong response (iteration {})", addr, loop_iteration); - break; - } - } - continue; // Don't forward ping to client - } - NetworkMessage::Pong(nonce) => { - // Handle pong directly - let mut peer_guard = peer.write().await; - if let Err(e) = peer_guard.handle_pong(*nonce) { - tracing::error!("Failed to handle pong from {}: {}", addr, e); - } - continue; // Don't forward pong to client - } - NetworkMessage::Version(_) | NetworkMessage::Verack => { - // These are handled during handshake, ignore here - tracing::trace!( - "Ignoring handshake message {:?} from {}", - msg.cmd(), - addr - ); - continue; - } - NetworkMessage::Addr(addresses) => { - // Convert legacy addr messages to AddrV2 format - let converted: Vec = addresses - .iter() - .filter_map(|(time, a)| { - let socket = a.socket_addr().ok()?; - let addr_v2 = match socket.ip() { - std::net::IpAddr::V4(v4) => AddrV2::Ipv4(v4), - std::net::IpAddr::V6(v6) => AddrV2::Ipv6(v6), - }; - Some(AddrV2Message { - time: *time, - services: a.services, - addr: addr_v2, - port: socket.port(), - }) - }) - .collect(); - if !converted.is_empty() { - tracing::debug!( - "Converted {} legacy addr entries from {}", - converted.len(), - addr - ); - addrv2_handler.handle_addrv2(converted).await; - } - continue; - } - NetworkMessage::Headers(headers) => { - // Log headers messages specifically - tracing::info!( - "📨 Received Headers message from {} with {} headers! (regular uncompressed)", - addr, - headers.len() - ); - // Check if peer supports headers2 - let peer_guard = peer.read().await; - if peer_guard.supports_headers2() { - tracing::warn!("⚠️ Peer {} supports headers2 but sent regular headers - possible protocol issue", addr); - } - drop(peer_guard); - // Forward to client - } - NetworkMessage::Headers2(headers2) => { - // Decompress headers in network layer and forward as regular Headers - tracing::info!( - "Received Headers2 from {} with {} compressed headers - decompressing", - addr, - headers2.headers.len() - ); - - match headers2_state.process_headers(&headers2.headers) { - Ok(headers) => { - tracing::info!( - "Decompressed {} headers from {} - forwarding as regular Headers", - headers.len(), - addr - ); - // Forward as regular Headers message - let headers_msg = NetworkMessage::Headers(headers); - let message = Message::new(msg.peer_address(), headers_msg); - message_dispatcher.lock().await.dispatch(&message); - continue; // Already sent, don't forward the original Headers2 - } - Err(e) => { - tracing::error!( - "Headers2 decompression failed from {}: {} - disabling headers2", - addr, - e - ); - headers2_disabled.lock().await.insert(addr); - // Apply reputation penalty - reputation_manager - .update_reputation( - addr, - ChangeReason::Headers2DecompressionFailed, - ) - .await; - continue; // Don't forward corrupted message - } - } - } - NetworkMessage::GetHeaders(_) => { - // SPV clients don't serve headers to peers - tracing::debug!( - "Received GetHeaders from {} - ignoring (SPV client)", - addr - ); - continue; // Don't forward to client - } - NetworkMessage::GetHeaders2(_) => { - // SPV clients don't serve compressed headers to peers - tracing::debug!( - "Received GetHeaders2 from {} - ignoring (SPV client)", - addr - ); - continue; // Don't forward to client - } - NetworkMessage::Unknown { - command, - payload, - } => { - // Log unknown messages with more detail - tracing::warn!("Received unknown message from {}: command='{}', payload_len={}", - addr, command, payload.len()); - // Still forward to client - } - _ => { - // Forward other messages to client - tracing::trace!( - "Forwarding {:?} from {} to client", - msg.cmd(), - addr - ); - } - } + pub fn events(&self) -> broadcast::Receiver { + self.events_tx.subscribe() + } +} - message_dispatcher.lock().await.dispatch(&msg); - } - Ok(None) => { - // No message available, continue immediately - // The socket read timeout already provides necessary delay - continue; - } - Err(e) => { - match e { - NetworkError::PeerDisconnected => { - tracing::info!("Peer {} disconnected", addr); - break; - } - NetworkError::Timeout => { - tracing::debug!("Timeout reading from {}, continuing...", addr); - // Minor reputation penalty for timeout - reputation_manager - .update_reputation(addr, ChangeReason::ReadTimeout) - .await; - continue; - } - _ => { - tracing::error!("Fatal error reading from {}: {}", addr, e); - - // Check if this is a serialization error that might have context - if let NetworkError::Serialization(ref decode_error) = e { - let error_msg = decode_error.to_string(); - if error_msg.contains("unknown special transaction type") { - tracing::warn!("Peer {} sent block with unsupported transaction type: {}", addr, decode_error); - tracing::error!( - "BLOCK DECODE FAILURE - Error details: {}", - error_msg - ); - // Reputation penalty for invalid data - reputation_manager - .update_reputation( - addr, - ChangeReason::InvalidTransactionInBlock, - ) - .await; - } else if error_msg - .contains("Failed to decode transactions for block") - { - // The error now includes the block hash - tracing::error!("Peer {} sent block that failed transaction decoding: {}", addr, decode_error); - // Try to extract the block hash from the error message - if let Some(hash_start) = error_msg.find("block ") { - if let Some(hash_end) = - error_msg[hash_start + 6..].find(':') - { - let block_hash = &error_msg - [hash_start + 6..hash_start + 6 + hash_end]; - tracing::error!( - "FAILING BLOCK HASH: {}", - block_hash - ); - } - } - } else if error_msg.contains("IO error") { - // This might be our wrapped error - log it prominently - tracing::error!("BLOCK DECODE FAILURE - IO error (possibly unknown transaction type) from peer {}", addr); - tracing::error!( - "Serialization error from {}: {}", - addr, - decode_error - ); - } else { - tracing::error!( - "Serialization error from {}: {}", - addr, - decode_error - ); - } - } - - break; - } - } - } +#[allow(clippy::too_many_arguments)] +fn spawn_router( + queue: Arc, + connected: Arc>>, + others: Arc>>, + inbound: UnboundedSender, + shutdown: CancellationToken, + bytes: Arc, + global_cap: Arc, + events: broadcast::Sender, +) -> JoinHandle<()> { + tokio::spawn(async move { + loop { + if shutdown.is_cancelled() { + break; + } + // Wait for work. `notify` fires both when a message is queued and + // when a response frees a peer slot. + if queue.len() == 0 { + tokio::select! { + _ = shutdown.cancelled() => break, + _ = queue.notify.notified() => continue, } } - // Remove from pool and notify consumers - tracing::warn!("Disconnecting from {} (peer reader loop ended)", addr); - Self::remove_peer_and_notify( - &pool, - &addr, - &connected_peer_count, - &network_event_sender, - ) - .await; - - headers2_disabled.lock().await.remove(&addr); - - // Give small positive reputation if peer maintained long connection - let conn_duration = Duration::from_secs(60 * loop_iteration); // Rough estimate - if conn_duration > Duration::from_secs(3600) { - // 1 hour - reputation_manager.update_reputation(addr, ChangeReason::LongUptime).await; + // Auto-connect more peers while the send queue is backed up: a deep + // queue means the connected peers can't serve our requests as fast as + // the pipelines produce them, so recruiting more raises the aggregate + // serving capacity and drains it — pushing the bottleneck toward OUR + // download link. Growth is bounded by `MAX_CONNECTED_PEERS`. Safe to + // grow past the initial `max_peers` now that each peer self-throttles + // to its MEASURED cap (a slow newcomer gets a tiny cap instead of + // feeding the old over-commit spiral). + if queue.len() > HIGH_DEMAND_QUEUE && connected.lock().await.len() < MAX_CONNECTED_PEERS + { + add_peers( + &connected, + &others, + &inbound, + &shutdown, + &bytes, + &events, + ADD_PEERS_BATCH, + MAX_CONNECTED_PEERS, + ) + .await; } - }); - } - /// Start the request processing task for outgoing messages from managers via RequestSender. - async fn start_request_processor(&self) { - // Take the receiver (only one task can own it) - let request_rx = { - let mut rx_guard = self.request_rx.lock().await; - rx_guard.take() - }; - - let Some(mut request_rx) = request_rx else { - tracing::warn!("Request processor already started or receiver unavailable"); - return; - }; - - let this = self.clone(); - let shutdown_token = self.shutdown_token.clone(); - - let mut tasks = self.tasks.lock().await; - tasks.spawn(async move { - tracing::info!("Starting request processor task"); - loop { + let peers = connected.lock().await; + let sent = + route_tick(&queue, &peers, global_cap.load(Ordering::Relaxed), &events).await; + drop(peers); + + if sent == 0 { + // Queue non-empty but every peer is at its in-flight cap: wait for + // a response to free capacity (the pump notifies on each response) + // — but never wait forever, because some of those "in-flight" + // requests may be dead. A peer that stops answering keeps its + // in-flight slots raised (the pipeline times the request out and + // re-queues it, but the peer is never told), so with enough dead + // requests every peer looks full, no response is coming, and the + // router would sleep on a notify that never fires — a hard sync + // stall with a backed-up queue. On the timeout we reclaim those + // slots and drop the peers that leaked them: they have had far + // longer than the pipelines' own timeout to answer, and leaving + // them connected would just route the freed work straight back to + // the peer that ignored it (the router picks the emptiest peer). tokio::select! { - request = request_rx.recv() => { - match request { - Some(NetworkRequest::SendMessage(msg)) => { - tracing::debug!("Request processor: sending {}", msg.cmd()); - // Spawn each send concurrently to allow parallel requests across peers. - let this = this.clone(); - tokio::spawn(async move { - let result = match &msg { - // Distribute across peers for parallel sync - NetworkMessage::GetCFHeaders(_) - | NetworkMessage::GetCFilters(_) - | NetworkMessage::GetData(_) - | NetworkMessage::GetMnListD(_) - | NetworkMessage::GetQRInfo(_) - | NetworkMessage::GetHeaders(_) - | NetworkMessage::GetHeaders2(_) => { - this.send_distributed(msg).await - } - _ => { - this.send_to_single_peer(msg).await - } - }; - if let Err(e) = result { - tracing::error!("Request processor: failed to send message: {}", e); - } - }); - } - Some(NetworkRequest::SendMessageToPeer(msg, peer_address)) => { - tracing::debug!("Request processor: sending {} to peer {}", msg.cmd(), peer_address); - let this = this.clone(); - tokio::spawn(async move { - let fallback_msg = msg.clone(); - let result = match this.pool.get_peer(&peer_address).await { - Some(peer) => match this.send_message_to_peer(&peer_address, &peer, msg).await { - Ok(()) => Ok(()), - Err(err) => { - tracing::warn!( - "Target peer {} send failed ({}), falling back to distributed send", - peer_address, - err - ); - this.send_distributed(fallback_msg).await - } - }, - None => { - tracing::warn!( - "Target peer {} disconnected, falling back to distributed send", - peer_address - ); - this.send_distributed(fallback_msg).await - } - }; - if let Err(e) = result { - tracing::error!("Request processor: failed to send message to peer {}: {}", peer_address, e); - } - }); - } - Some(NetworkRequest::BroadcastMessage(msg)) => { - tracing::debug!("Request processor: broadcasting {}", msg.cmd()); - let this = this.clone(); - tokio::spawn(async move { - let results = this.broadcast(msg).await; - let failures = results.iter().filter(|r| r.is_err()).count(); - if failures > 0 { - tracing::warn!( - "Request processor: broadcast had {} failures out of {} peers", - failures, - results.len() - ); - } - }); + _ = shutdown.cancelled() => break, + _ = queue.notify.notified() => {}, + _ = tokio::time::sleep(STALL_CHECK) => { + let mut peers = connected.lock().await; + let before = peers.len(); + let mut reclaimed = 0; + // Manual drain/rebuild rather than `Vec::retain`: reaping now + // awaits the per-peer latency lock, and a `retain` closure is + // synchronous. + let mut kept = Vec::with_capacity(peers.len()); + for (p, s) in peers.drain(..) { + let n = p.reap_stale(STALE_REQUEST).await; + if n == 0 { + kept.push((p, s)); + continue; } - None => { - tracing::info!("Request processor: channel closed"); - break; + reclaimed += n; + // Drop the peer only if it has NEVER answered anything — + // that one is dead weight, and leaving it connected would + // route the freed work straight back to it (the router + // picks the emptiest peer). A peer that HAS served us is + // merely slow: a 2MB block over a weak link legitimately + // takes a while, and dropping those cost us 14 of 16 + // peers mid-sync. We only wanted the slot back. + let (completed, _) = p.latency_totals(); + if completed == 0 { + p.close(); + } else { + kept.push((p, s)); } } - } - _ = shutdown_token.cancelled() => { - tracing::info!("Request processor: shutting down"); - break; - } - } - } - }); - } - - pub(crate) async fn evict_mismatched_peers(&self) { - if self.required_services == ServiceFlags::NONE { - return; - } - let all_peers = self.pool.get_all_peers().await; - let connected_count = all_peers.len(); - if connected_count <= 1 { - return; - } - let mut matched_count = 0; - let mut mismatched = Vec::new(); - for (addr, peer) in &all_peers { - let peer_guard = peer.read().await; - if peer_guard.services_known() && peer_guard.has_service(self.required_services) { - matched_count += 1; - } else if peer_guard.services_known() { - mismatched.push(*addr); - } - } - if mismatched.is_empty() { - return; - } - let drop_count = if matched_count > 0 { - mismatched.len() - } else { - mismatched.len().min(connected_count - 1) - }; - if drop_count == 0 { - return; - } - tracing::info!( - "Capability churn: dropping {} of {} peers lacking required services", - drop_count, - connected_count, - ); - for addr in mismatched.into_iter().take(drop_count) { - self.record_capability_rejection(addr).await; - let _ = self - .disconnect_peer( - &addr, - &format!("missing required services ({})", self.required_services), - ) - .await; - } - } - - async fn maintenance_tick(&self) { - // Remove peers that the reader loop failed to clean up. - // This should not trigger under normal operation. - let unhealthy = self.pool.remove_unhealthy().await; - for addr in &unhealthy { - tracing::warn!("Maintenance removed stale peer {} - reader loop missed cleanup", addr); - Self::notify_peer_removed( - &self.pool, - addr, - &self.connected_peer_count, - &self.network_event_sender, - ) - .await; - } - - let count = self.pool.peer_count().await; - tracing::debug!("Connected peers: {}", count); - // Keep the cached counter in sync with actual pool count - self.connected_peer_count.store(count, Ordering::Relaxed); - if self.exclusive_mode { - // In exclusive mode, only reconnect to originally specified peers - for addr in self.initial_peers.iter() { - if !self.pool.is_connected(addr).await && !self.pool.is_connecting(addr).await { - tracing::info!("Reconnecting to exclusive peer: {}", addr); - self.connect_to_peer(*addr).await; - } - } - } else { - // Evict peers that lack required services before top-up so replacements - // can be pulled in during the same tick. - self.evict_mismatched_peers().await; - // Re-read count after potential churn so top-up sees the current pool size. - let count = self.pool.peer_count().await; - if count < self.max_peers { - // Try known addresses first, sorted by reputation - let known = self.addrv2_handler.get_known_addresses().await; - let needed = self.max_peers.saturating_sub(count); - // Select best peers based on reputation - let best_peers = self.reputation_manager.select_best_peers(known, needed * 2).await; - let mut attempted = 0; - - for addr in best_peers { - if self.is_capability_rejected(&addr).await { - continue; - } - if !self.pool.is_connected(&addr).await && !self.pool.is_connecting(&addr).await - { - self.connect_to_peer(addr).await; - attempted += 1; - if attempted >= needed { - break; + *peers = kept; + if reclaimed > 0 { + tracing::warn!( + target: "dash_spv::network", + "reclaimed {} stalled request slot(s); dropped {} never-responding peer(s) -> {} connected", + reclaimed, + before - peers.len(), + peers.len(), + ); } } } } } + }) +} - if self.shutdown_token.is_cancelled() { - return; - } - - // Send ping to all peers if needed and disconnect unresponsive ones - for (addr, peer) in self.pool.get_all_peers().await { - let mut peer_guard = peer.write().await; - if peer_guard.should_ping() { - if let Err(e) = peer_guard.send_ping().await { - tracing::error!("Failed to ping {}: {}", addr, e); - // Update reputation for ping failure - self.reputation_manager.update_reputation(addr, ChangeReason::PingFailed).await; - } - } - let has_expired = peer_guard.remove_expired_pings(); - drop(peer_guard); - if has_expired { - let _ = self.disconnect_peer(&addr, "ping timeout").await; - } - } - - // Only save known peers if not in exclusive mode - if !self.exclusive_mode { - let addresses = self.addrv2_handler.get_known_addresses().await; - if !addresses.is_empty() { - if let Err(e) = self.peer_store.save_peers(&addresses).await { - tracing::warn!("Failed to save peers: {}", e); - } - } - - // Save reputation data periodically - if let Err(e) = self.reputation_manager.save_to_storage(&*self.peer_store).await { - tracing::warn!("Failed to save reputation data: {}", e); - } +/// Extract the pipeline key of a router-paced request, so the router can report +/// it as on-flight. Returns `None` for non-pipeline messages. +fn request_keys(msg: &NetworkMessage) -> Vec { + match msg { + NetworkMessage::GetHeaders(m) | NetworkMessage::GetHeaders2(m) => { + m.locator_hashes.first().map(|h| RequestKey::Headers(*h)).into_iter().collect() } + NetworkMessage::GetCFHeaders(m) => vec![RequestKey::CfHeaders(m.stop_hash)], + NetworkMessage::GetCFilters(m) => vec![RequestKey::CFilters(m.start_height)], + NetworkMessage::GetMnListD(m) => vec![RequestKey::MnListDiff(m.block_hash)], + // One `getdata` may name several blocks; each is its own tracked request. + NetworkMessage::GetData(inv) => inv + .iter() + .filter_map(|i| match i { + Inventory::Block(h) => Some(RequestKey::Block(*h)), + _ => None, + }) + .collect(), + _ => Vec::new(), } +} - async fn dns_fallback_tick(&self) { - let count = self.pool.peer_count().await; - if count >= self.max_peers { - return; - } - let dns_peers = tokio::select! { - peers = self.discovery.discover_peers(self.network) => peers, - _ = self.shutdown_token.cancelled() => { - tracing::info!("Maintenance loop shutting down during DNS discovery"); - return - } - }; - let needed = self.max_peers.saturating_sub(count); - tracing::debug!("DNS fallback tick found {} addresses. Needed {}", dns_peers.len(), needed); - let mut dns_attempted = 0; - for addr in dns_peers.iter() { - if self.is_capability_rejected(addr).await { - continue; - } - if !self.pool.is_connected(addr).await && !self.pool.is_connecting(addr).await { - self.connect_to_peer(*addr).await; - dns_attempted += 1; - if dns_attempted >= needed { - break; - } - } - } +async fn route_tick( + queue: &MsgQueue, + peers: &[(ConnectedPeer, State)], + global_cap: usize, + events: &broadcast::Sender, +) -> usize { + if peers.is_empty() { + return 0; } - /// Start peer connection maintenance loop - async fn start_maintenance_loop(&self) { - let this = self.clone(); - let mut tasks = self.tasks.lock().await; - tasks.spawn(async move { - // Periodic DNS discovery check (only active in non-exclusive mode) - let mut dns_interval = - time::interval_at(Instant::now() + DNS_DISCOVERY_DELAY, DNS_DISCOVERY_DELAY); - // Periodic reconnection check (active in both modes) - let mut maintenance_interval = time::interval(MAINTENANCE_INTERVAL); - let mut network_events = this.network_event_sender.subscribe(); - while !this.shutdown_token.is_cancelled() { - tokio::select! { - _ = maintenance_interval.tick() => { - tracing::debug!("Maintenance interval elapsed"); - this.maintenance_tick().await; - } - _ = dns_interval.tick(), if !this.exclusive_mode => { - this.dns_fallback_tick().await; - } - event = network_events.recv() => { - match event { - Ok(event) => { - tracing::debug!("Network event in maintenance loop: {}", event); - dns_interval.reset(); - this.maintenance_tick().await; - } - Err(error) => { - tracing::error!("Network event error: {}", error); - break; - } - } - } - _ = this.shutdown_token.cancelled() => { - tracing::info!("Maintenance loop shutting down"); - break; - } - } - } - }); + // Free capacity this round = min(sum of per-peer room, global room). Each + // peer's cap is its MEASURED serving capacity (its bandwidth-delay product), + // sized by the controller from that peer's own completion rate and service + // time — fast peers carry more, slow peers less, with no fixed constant. The + // global cap is our measured download capacity. Whichever binds first limits + // this round, so we ride each peer's real ceiling without over-committing. + let total_in_flight: usize = peers.iter().map(|(p, _)| p.in_flight()).sum(); + let per_peer_room: usize = + peers.iter().map(|(p, _)| p.cap().saturating_sub(p.in_flight())).sum(); + let global_room = global_cap.saturating_sub(total_in_flight); + let capacity = per_peer_room.min(global_room); + if capacity == 0 { + return 0; } - /// Send a message to a single peer selected by message type requirements. - async fn send_to_single_peer(&self, message: NetworkMessage) -> NetworkResult<()> { - let peers = self.pool.get_all_peers().await; - - if peers.is_empty() { - return Err(NetworkError::ConnectionFailed("No connected peers".to_string())); - } - - let preferred_service = match &message { - NetworkMessage::FilterLoad(_) - | NetworkMessage::FilterClear - | NetworkMessage::MemPool => Some((ServiceFlags::BLOOM, true)), - NetworkMessage::GetCFHeaders(_) | NetworkMessage::GetCFilters(_) => { - Some((ServiceFlags::COMPACT_FILTERS, true)) - } - NetworkMessage::GetHeaders(_) | NetworkMessage::GetHeaders2(_) => { - Some((ServiceFlags::NODE_HEADERS_COMPRESSED, false)) - } - _ => None, + let msgs = queue.pop_n(capacity).await; + let mut sent = 0; + // Anything popped that we could not put on the wire goes BACK on the queue. + // Dropping it would strand the owning pipeline forever: it has already marked + // the request as handed to the network, and its response timeout only starts + // when the router reports the request on the wire, so a dropped message is + // never re-sent and never times out. + let mut unsent: Vec = Vec::new(); + let mut msgs = msgs.into_iter(); + for msg in msgs.by_ref() { + // Send to the peer with the most free measured capacity. + let Some((peer, _)) = peers + .iter() + .filter(|(p, _)| p.in_flight() < p.cap()) + .max_by_key(|(p, _)| p.cap().saturating_sub(p.in_flight())) + else { + unsent.push(msg); // every peer is at its measured cap + break; }; - - let (addr, peer) = if let Some((flags, required)) = preferred_service { - match self.pool.peer_with_service(flags).await { - Some((address, peer)) => { - tracing::debug!( - "Selected peer {} with {} for {}", - address, - flags, - message.cmd() - ); - (address, peer) - } - None if required => { - tracing::warn!("No peers support {}, cannot send {}", flags, message.cmd()); - return Err(NetworkError::ProtocolError(format!("No peers support {}", flags))); - } - None => self.next_peer(&peers), + if peer.send(&msg).await.is_ok() { + sent += 1; + // Tell the owning pipeline this request is now on the wire so it can + // start the response timeout from here, not from when it was queued. + for key in request_keys(&msg) { + let _ = events.send(NetworkEvent::RequestOnFlight(key)); } } else { - self.next_peer(&peers) - }; - - self.send_message_to_peer(&addr, &peer, message).await - } - - /// Send a message distributed across connected peers using round-robin selection. - /// - /// Peer selection and message handling based on message type: - /// - Filters (GetCFHeaders/GetCFilters): requires peers that support compact filters - /// - Headers (GetHeaders/GetHeaders2): prefers headers2 peers, upgrades GetHeaders if supported - /// - Other (blocks, masternode data, etc.): uses all connected peers - async fn send_distributed(&self, message: NetworkMessage) -> NetworkResult<()> { - let peers = self.pool.get_all_peers().await; - - if peers.is_empty() { - return Err(NetworkError::ConnectionFailed("No connected peers".to_string())); + tracing::warn!(target: "dash_spv::network", "router: send to {} failed", peer.addr()); + unsent.push(msg); } + } + unsent.extend(msgs); // whatever the loop never reached + queue.push_front_all(unsent).await; + + if sent > 0 { + tracing::debug!( + target: "dash_spv::network", + "router: sent {} | peers={} queue={}", + sent, + peers.len(), + queue.len(), + ); + } + sent +} - // Select eligible peers based on message type - let (selected_peers, require_capability) = match &message { - NetworkMessage::GetCFHeaders(_) | NetworkMessage::GetCFilters(_) => { - let filter_peers = - self.pool.peers_with_service(ServiceFlags::COMPACT_FILTERS).await; - (filter_peers, true) +/// Sizes the global in-flight budget from MEASURED download capacity — no fixed +/// magic number, per the design goal of estimating how many requests the current +/// network can absorb before it saturates. +/// +/// The estimate is Little's Law applied to downloads: the number of requests in +/// flight that sustains a completion rate `λ` at an uncongested per-request +/// service time `W` is `L = λ · W`. We measure both from the peers' response +/// stream — `λ` = requests completed per second, `W` = average time from send to +/// the response that completes the request (for `getcfilters`, dominated by the +/// download time of its ~1000 `cfilter`s, i.e. our downlink). We track the +/// minimum `W` as the uncongested baseline (the pipe's true latency at the front +/// of the knee) and target `L = λ · min_W`, probing slightly past it. +/// +/// Saturation is detected by `W` INFLATION, not by throughput: once in-flight +/// exceeds the bandwidth-delay product, extra requests just queue at the peers, +/// so `W` climbs while `λ` (and the download rate) plateaus. That inflation is +/// visible even though a raw throughput meter can't see the knee (a prior +/// throughput hill-climb ran away and regressed sync ~8x because the backlog +/// kept bytes flowing). When `W > min_W · INFLATE` we stop probing and shrink +/// back toward the sustaining level, so we ride just below saturation. +fn spawn_bandwidth_controller( + bytes: Arc, + cap: Arc, + connected: Arc>>, + shutdown: CancellationToken, +) -> JoinHandle<()> { + const WINDOW: Duration = Duration::from_millis(500); + const FLOOR_PER_PEER: usize = 2; // global floor = peers · this + const PEER_CEIL: usize = 32; // per-connection sanity bound on in-flight + const RISE: f64 = 1.05; // throughput must climb 5% to justify a bigger cap + const DROP: f64 = 0.85; // throughput below this·last => over-commit, back off + const REPROBE: u32 = 8; // plateau windows to hold before nudging the cap up + const IDLE_BPS: f64 = 1.0e6; // downlink under 1 MB/s = idle, hold the cap + const EMA_ALPHA: f64 = 0.5; // smoothing for the noisy per-window rate + let window_s = WINDOW.as_secs_f64(); + + tokio::spawn(async move { + let mut last_bytes = bytes.load(Ordering::Relaxed); + let mut rate_ema = 0.0f64; // smoothed downlink bytes/s + let mut last_rate = 0.0f64; // smoothed rate at the previous cap adjustment + let mut hold = 0u32; // consecutive plateau windows + let mut ticker = tokio::time::interval(WINDOW); + loop { + tokio::select! { + _ = shutdown.cancelled() => break, + _ = ticker.tick() => {} } - NetworkMessage::GetHeaders(_) | NetworkMessage::GetHeaders2(_) => { - // Prefer headers2 peers (excluding disabled), fall back to all - let disabled = self.headers2_disabled.lock().await; - let mut headers2_peers = - self.pool.peers_with_service(ServiceFlags::NODE_HEADERS_COMPRESSED).await; - headers2_peers.retain(|(addr, _)| !disabled.contains(addr)); - drop(disabled); - if headers2_peers.is_empty() { - (peers.clone(), false) - } else { - (headers2_peers, false) - } - } - _ => { - // All other messages use all connected peers - (peers.clone(), false) - } - }; - if selected_peers.is_empty() { - return if require_capability { - Err(NetworkError::ProtocolError("No peers support required capability".to_string())) + // Downlink throughput (bytes read off the sockets — our downlink only). + // This is the saturation signal: unlike per-request service time — which + // balloons with cfilter payload size and out-of-order batch completion, + // reading "saturated" forever and pinning the cap at its floor — bytes/s + // directly reflects whether we are using the pipe. We grow the in-flight + // budget while throughput keeps climbing with it and stop at the plateau + // (the bandwidth-delay product: past it, more in-flight only grows queues, + // not bytes/s), backing off if it collapses (peers over-committed). + let now_bytes = bytes.load(Ordering::Relaxed); + let dl_rate = now_bytes.saturating_sub(last_bytes) as f64 / window_s; + last_bytes = now_bytes; + rate_ema = if rate_ema == 0.0 { + dl_rate } else { - Err(NetworkError::ConnectionFailed("No connected peers".to_string())) + EMA_ALPHA * dl_rate + (1.0 - EMA_ALPHA) * rate_ema }; - } - let (addr, peer) = self.next_peer(&selected_peers); - - tracing::debug!("Distributing {} request to peer {}", message.cmd(), addr); - - self.send_message_to_peer(&addr, &peer, message).await - } - - /// Pick the next peer from `peers` using round-robin rotation. - fn next_peer( - &self, - peers: &[(SocketAddr, Arc>)], - ) -> (SocketAddr, Arc>) { - let idx = self.round_robin_counter.fetch_add(1, Ordering::Relaxed) % peers.len(); - (peers[idx].0, peers[idx].1.clone()) - } - - /// Send a message to the given peer. - /// For GetHeaders messages upgrade to GetHeaders2 if the peer supports it. - async fn send_message_to_peer( - &self, - addr: &SocketAddr, - peer: &Arc>, - message: NetworkMessage, - ) -> NetworkResult<()> { - let message = match message { - NetworkMessage::GetHeaders(get_headers) => { - let supports_headers2 = peer.read().await.can_request_headers2(); - if supports_headers2 && !self.headers2_disabled.lock().await.contains(addr) { - tracing::debug!("Upgrading GetHeaders to GetHeaders2 for peer {}", addr); - NetworkMessage::GetHeaders2(get_headers) + let npeers = connected.lock().await.len(); + if npeers == 0 { + continue; + } + let floor = (npeers * FLOOR_PER_PEER).max(FLOOR_PER_PEER); + let ceiling = (npeers * PEER_CEIL).max(floor + 1); + let step = npeers.max(4); // ~one extra slot per peer per window + let cur = cap.load(Ordering::Relaxed); + + // Gradient hill-climb on smoothed throughput. + let (new, action) = if rate_ema < IDLE_BPS { + // Nothing meaningful downloading (e.g. the commit tail): hold the + // budget steady so it is ready when the download resumes. + (cur, "idle") + } else if cur <= floor || rate_ema >= last_rate * RISE { + // Still gaining (or at the floor): push the budget up. + hold = 0; + ((cur + step).min(ceiling), "grow") + } else if rate_ema < last_rate * DROP { + // Throughput collapsed — the peers are over-committed. Back off. + hold = 0; + (((cur as f64 * 0.8) as usize).max(floor), "backoff") + } else { + // Plateau: we are at the knee. Hold, re-probing up occasionally to + // catch a capacity increase (a faster peer, less congestion). + hold += 1; + if hold >= REPROBE { + hold = 0; + ((cur + step).min(ceiling), "reprobe") } else { - NetworkMessage::GetHeaders(get_headers) + (cur, "hold") } + }; + // Anchor RISE/DROP to the rate at each real adjustment (skip idle/hold + // windows) so the next comparison is like-for-like. + if matches!(action, "grow" | "backoff" | "reprobe") { + last_rate = rate_ema; } - other => other, - }; - - let mut peer_guard = peer.write().await; - peer_guard - .send_message(message) - .await - .map_err(|e| NetworkError::ProtocolError(format!("Failed to send to {}: {}", addr, e))) - } - - /// Broadcast a message to all connected peers - pub async fn broadcast(&self, message: NetworkMessage) -> Vec> { - let peers = self.pool.get_all_peers().await; - let mut handles = Vec::new(); - - // Spawn tasks for concurrent sending - for (addr, peer) in peers { - // Reduce verbosity for common sync messages - match &message { - NetworkMessage::GetHeaders(_) | NetworkMessage::GetCFilters(_) => { - tracing::debug!("Broadcasting {} to {}", message.cmd(), addr); - } - _ => { - tracing::trace!("Broadcasting {:?} to {}", message.cmd(), addr); + cap.store(new, Ordering::Relaxed); + + // Spread the global budget evenly across peers with a slot of headroom, + // so the GLOBAL cap is the binding constraint (route_tick fills the + // emptiest peer first, so fast peers naturally serve more) while no + // single connection is over-committed beyond PEER_CEIL. + let per_peer = (new.div_ceil(npeers) + 1).clamp(FLOOR_PER_PEER, PEER_CEIL); + { + let g = connected.lock().await; + for (p, _) in g.iter() { + p.set_cap(per_peer); } } - let msg = message.clone(); - let handle = tokio::spawn(async move { - let mut peer_guard = peer.write().await; - peer_guard.send_message(msg).await.map_err(Error::Network) - }); - handles.push(handle); + tracing::debug!( + target: "dash_spv::network", + "bandwidth: {:.1} MB/s (ema {:.1}) | {} | global cap={} | peers={} per-peer cap={}", + dl_rate / 1e6, + rate_ema / 1e6, + action, + new, + npeers, + per_peer, + ); } + }) +} - // Wait for all sends to complete - let mut results = Vec::new(); - for handle in handles { - match handle.await { - Ok(result) => results.push(result), - Err(_) => results.push(Err(Error::Network(NetworkError::ConnectionFailed( - "Task panicked during broadcast".to_string(), - )))), +/// Keep the peer set topped up. +/// +/// Peers are connected once, in `start`; nothing put them back afterwards. The router's +/// `add_peers` only fires under backpressure (a deep send queue), so a client whose peers +/// all dropped while it was idle — or mid-sync with a short queue — simply sat there with +/// zero peers forever. This watches the count and refills it, pulling fresh candidates +/// from the discoverer when the backup list runs dry. +#[allow(clippy::too_many_arguments)] +fn spawn_reconnector( + discoverer: Arc>, + connected: Arc>>, + others: Arc>>, + inbound: UnboundedSender, + shutdown: CancellationToken, + bytes: Arc, + events: broadcast::Sender, + best_tip: Arc, + max_peers: usize, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(RECONNECT_CHECK); + + loop { + tokio::select! { + _ = shutdown.cancelled() => break, + _ = ticker.tick() => {} } - } - - results - } - - /// Disconnect a specific peer - pub async fn disconnect_peer(&self, addr: &SocketAddr, reason: &str) -> Result<(), Error> { - tracing::info!("Disconnecting peer {} - reason: {}", addr, reason); - Self::remove_peer_and_notify( - &self.pool, - addr, - &self.connected_peer_count, - &self.network_event_sender, - ) - .await; - - Ok(()) - } - - /// Get reputation information for all peers - pub async fn get_peer_reputations(&self) -> HashMap { - let reputations = self.reputation_manager.get_all_reputations().await; - reputations.into_iter().map(|(addr, rep)| (addr, (rep.score, rep.is_banned()))).collect() - } - - /// Ban a specific peer manually - pub async fn ban_peer(&self, addr: &SocketAddr, reason: &str) -> Result<(), Error> { - tracing::info!("Manually banning peer {} - reason: {}", addr, reason); - - // Disconnect the peer first - self.disconnect_peer(addr, reason).await?; - - // Update reputation to trigger ban - self.reputation_manager.update_reputation(*addr, ChangeReason::ManuallyBanned).await; - - Ok(()) - } + let deficit = max_peers.saturating_sub(connected.lock().await.len()); + if deficit == 0 { + continue; + } - /// Unban a specific peer - pub async fn unban_peer(&self, addr: &SocketAddr) { - self.reputation_manager.unban_peer(addr).await; - } + // Backups first (already discovered, not yet used); otherwise go ask for more. + if others.lock().await.is_empty() { + let fresh = discoverer.lock().await.get(deficit.max(ADD_PEERS_BATCH)).await; - /// Shutdown the network manager - pub async fn shutdown(&self) { - tracing::info!("Shutting down peer network manager"); - self.shutdown_token.cancel(); + // The discoverer hands out of a pool it keeps, so it can name a peer we are + // already talking to. Filter those out rather than opening a second socket + // to the same node. + let live: HashSet = + connected.lock().await.iter().map(|(peer, _)| peer.addr()).collect(); + let fresh: Vec = + fresh.into_iter().filter(|p| !live.contains(&p.addr())).collect(); - // Save known peers before shutdown - let addresses = self.addrv2_handler.get_addresses_for_peer(MAX_ADDR_TO_STORE).await; - if !addresses.is_empty() { - if let Err(e) = self.peer_store.save_peers(&addresses).await { - tracing::warn!("Failed to save peers on shutdown: {}", e); + if fresh.is_empty() { + continue; + } + others.lock().await.extend(fresh); } - } - // Save reputation data before shutdown - if let Err(e) = self.reputation_manager.save_to_storage(&*self.peer_store).await { - tracing::warn!("Failed to save reputation data on shutdown: {}", e); - } + let before = connected.lock().await.len(); + add_peers( + &connected, &others, &inbound, &shutdown, &bytes, &events, deficit, max_peers, + ) + .await; + let after = connected.lock().await.len(); - // Drain tasks while holding the lock. connect_to_peer() already uses - // `select!` with the cancellation token when acquiring this lock, so no - // deadlock can occur once the shutdown token is cancelled above. - let mut tasks = self.tasks.lock().await; - while let Some(result) = tasks.join_next().await { - if let Err(e) = result { - tracing::error!("Task join error: {}", e); + if after > before { + tracing::info!( + target: "dash_spv::network", + "reconnected: {} -> {} peers", + before, + after, + ); + // Wake the managers: those that stopped for lack of peers restart, and the + // mempool re-arms relay on the new peers. + let _ = events.send(NetworkEvent::PeersUpdated { + connected_count: after as u32, + best_height: best_tip.load(Ordering::Relaxed), + }); } } + }) +} - // Disconnect all peers - for addr in self.pool.get_connected_addresses().await { - self.pool.remove_peer(&addr).await; - } - } - - async fn record_capability_rejection(&self, addr: SocketAddr) { - Self::record_capability_rejection_in(&self.capability_rejected, addr).await; - } - - async fn is_capability_rejected(&self, addr: &SocketAddr) -> bool { - let mut rejected = self.capability_rejected.write().await; - let now = Instant::now(); - rejected.retain(|_, rejected_at| { - now.saturating_duration_since(*rejected_at) < CAPABILITY_REJECTED_TTL - }); - rejected.contains_key(addr) +#[allow(clippy::too_many_arguments)] +async fn add_peers( + connected: &Mutex>, + others: &Mutex>, + inbound: &UnboundedSender, + shutdown: &CancellationToken, + bytes: &Arc, + events: &broadcast::Sender, + batch: usize, + max_peers: usize, +) { + // Never grow past the operating cap; only backfill the deficit toward it. + let deficit = max_peers.saturating_sub(connected.lock().await.len()); + if deficit == 0 { + return; } - - async fn record_capability_rejection_in( - capability_rejected: &RwLock>, - addr: SocketAddr, - ) { - capability_rejected.write().await.insert(addr, Instant::now()); + let candidates: Vec = { + let mut others = others.lock().await; + let take = batch.min(deficit).min(others.len()); + others.drain(..take).collect() + }; + + let mut added = 0; + for candidate in candidates { + if let Ok(peer) = candidate.connect(inbound.clone(), shutdown.clone(), bytes.clone()).await + { + let addr = peer.addr(); + connected.lock().await.push((peer, State {})); + let _ = events.send(NetworkEvent::PeerConnected(addr)); + added += 1; + } } - async fn should_reject_after_handshake( - pool: &PeerPool, - peer: &Peer, - required_services: ServiceFlags, - ) -> bool { - required_services != ServiceFlags::NONE - && pool.has_peers_with_service(required_services).await - && peer.services_known() - && !peer.has_service(required_services) + if added > 0 { + let total = connected.lock().await.len(); + tracing::info!( + target: "dash_spv::network", + "high demand: added {} peers -> {} connected", + added, + total, + ); } } -// Implement Clone for use in async closures -impl Clone for PeerNetworkManager { - fn clone(&self) -> Self { - Self { - pool: self.pool.clone(), - max_peers: self.max_peers, - discovery: self.discovery.clone(), - addrv2_handler: self.addrv2_handler.clone(), - peer_store: self.peer_store.clone(), - reputation_manager: self.reputation_manager.clone(), - network: self.network, - shutdown_token: self.shutdown_token.clone(), - tasks: self.tasks.clone(), - initial_peers: self.initial_peers.clone(), - data_dir: self.data_dir.clone(), - user_agent: self.user_agent.clone(), - exclusive_mode: self.exclusive_mode, - required_services: self.required_services, - capability_rejected: self.capability_rejected.clone(), - connected_peer_count: self.connected_peer_count.clone(), - headers2_disabled: self.headers2_disabled.clone(), - message_dispatcher: self.message_dispatcher.clone(), - request_tx: self.request_tx.clone(), - request_rx: self.request_rx.clone(), - round_robin_counter: self.round_robin_counter.clone(), - network_event_sender: self.network_event_sender.clone(), +async fn probe_and_select( + discoverer: &mut PeerDiscoverer, + connected: &Mutex>, + others: &Mutex>, + max_peers: usize, + inbound: &UnboundedSender, + shutdown: &CancellationToken, + bytes: &Arc, +) -> u32 { + const CONNECT_CHUNK: usize = 16; // bounded concurrent handshakes per round + const FAST_LAG_MS: u32 = 100; // prefer peers that ping under this + + // Sort key for latency: lower is better; treat an unmeasured lag (0) as worst. + fn lag_key(p: &ConnectedPeer) -> u32 { + match p.lag_ms() { + 0 => u32::MAX, + ms => ms, } } -} -// Implement NetworkManager trait -#[async_trait] -impl NetworkManager for PeerNetworkManager { - async fn message_receiver(&mut self, types: &[MessageType]) -> UnboundedReceiver { - self.message_dispatcher.lock().await.message_receiver(types) + let mut candidates = discoverer.get(PROBE_BATCH).await; + // Cap how many candidates we handshake before settling. This bounds startup: + // real peers often ping over 100ms, so without a limit the "wait for fast + // peers" loop would probe the entire batch. We try a few times `max_peers` + // and then take the lowest-latency of whatever connected. + let attempt_max = (max_peers * 3).max(CONNECT_CHUNK * 2); + let mut attempted = 0usize; + let mut kept: Vec = Vec::new(); + let mut fallback: Vec = Vec::new(); // connected, not fast enough + let mut tip = 0u32; + + // Connect a chunk at a time and STOP as soon as we have `max_peers` fast + // peers (or we've tried enough), so sync starts without waiting on the whole + // probe batch. We keep only the peers we'll actually query connected; + // everything else stays a bare address for on-demand recruitment. + while kept.len() < max_peers && !candidates.is_empty() && attempted < attempt_max { + let take = CONNECT_CHUNK.min(candidates.len()); + attempted += take; + let batch: Vec = candidates.drain(..take).collect(); + let results = join_all( + batch.into_iter().map(|c| c.connect(inbound.clone(), shutdown.clone(), bytes.clone())), + ) + .await; + for peer in results.into_iter().filter_map(Result::ok) { + tip = tip.max(peer.version().start_height.max(0) as u32); + let lag = peer.lag_ms(); + if kept.len() < max_peers && lag > 0 && lag < FAST_LAG_MS { + kept.push(peer); + } else { + fallback.push(peer); + } + } } - fn request_sender(&self) -> RequestSender { - PeerNetworkManager::request_sender(self) + // If too few peers were fast, fill up to `max_peers` with the lowest-latency + // fallbacks; close and remember the rest as backups. + fallback.sort_by_key(lag_key); + while kept.len() < max_peers && !fallback.is_empty() { + kept.push(fallback.remove(0)); } - - async fn connect(&mut self) -> NetworkResult<()> { - self.start().await.map_err(|e| NetworkError::ConnectionFailed(e.to_string())) + let mut backups: Vec = candidates; // un-probed addresses + for peer in fallback { + peer.close(); // drop the connection we won't use + backups.push(peer.disconnect()); } - async fn disconnect(&mut self) -> NetworkResult<()> { - self.shutdown().await; - Ok(()) - } + let fast = kept.iter().filter(|p| p.lag_ms() > 0 && p.lag_ms() < FAST_LAG_MS).count(); + tracing::info!( + target: "dash_spv::network", + "probe: connected {} peers ({} fast <{}ms), {} backups | tip={}", + kept.len(), + fast, + FAST_LAG_MS, + backups.len(), + tip, + ); + + *connected.lock().await = kept.into_iter().map(|peer| (peer, State {})).collect(); + *others.lock().await = backups; + + tip +} - async fn send_message(&mut self, message: NetworkMessage) -> NetworkResult<()> { - // For sync messages that require consistent responses, send to only one peer - match &message { - NetworkMessage::GetHeaders(_) - | NetworkMessage::GetHeaders2(_) - | NetworkMessage::GetCFHeaders(_) - | NetworkMessage::GetCFilters(_) - | NetworkMessage::GetData(_) - | NetworkMessage::GetMnListD(_) => self.send_to_single_peer(message).await, - _ => { - // For other messages, broadcast to all peers - let results = self.broadcast(message).await; - - // Return error if all sends failed - if results.is_empty() { - return Err(NetworkError::ConnectionFailed("No connected peers".to_string())); - } +fn spawn_pump( + mut inbound: UnboundedReceiver, + subscribers: Subscribers, + connected: Arc>>, + events: broadcast::Sender, + queue: Arc, + shutdown: CancellationToken, +) -> JoinHandle<()> { + tokio::spawn(async move { + // Per-peer received-message counter to check load balance across peers. + let mut recv_by_peer: HashMap = HashMap::new(); + let mut total_recv: u64 = 0; + loop { + let t_idle = std::time::Instant::now(); + let event = tokio::select! { + _ = shutdown.cancelled() => break, + ev = inbound.recv() => match ev { + Some(ev) => ev, + None => break, + }, + }; + crate::timer::P_PUMP_IDLE.add(t_idle.elapsed()); + match event { + PeerEvent::Message(addr, msg) => { + let t_busy = std::time::Instant::now(); + *recv_by_peer.entry(addr).or_insert(0) += 1; + total_recv += 1; + if total_recv.is_multiple_of(250_000) { + let mut dist: Vec<(SocketAddr, u64)> = + recv_by_peer.iter().map(|(a, c)| (*a, *c)).collect(); + dist.sort_by_key(|(_, c)| std::cmp::Reverse(*c)); + tracing::info!( + target: "filt_depth", + "[t+{}ms] recv balance ({} peers, {} total): {:?}", + crate::timer::elapsed_ms(), + dist.len(), + total_recv, + dist, + ); + // Per-peer request->response latency (avg / worst). + let lat: Vec<(SocketAddr, u64, String)> = connected + .lock() + .await + .iter() + .map(|(p, _)| { + let (n, avg, max) = p.latency_stats(); + (p.addr(), n, format!("avg={avg:.1}ms max={max:.1}ms")) + }) + .collect(); + tracing::info!( + target: "filt_depth", + "[t+{}ms] peer latency: {:?}", + crate::timer::elapsed_ms(), + lat, + ); + } + let mt = MessageType::from_cmd(msg.cmd()); + // Single-message responses free a slot in the peer's reader; + // wake the router so it can use the freed capacity. `cfilter` + // is skipped — its batch frees a slot via `request_completed`, + // which notifies once per ~1000 messages instead of each. + if mt != Some(MessageType::CFilter) { + queue.notify.notify_one(); + } - let successes = results.iter().filter(|r| r.is_ok()).count(); - if successes == 0 { - return Err(NetworkError::ProtocolError( - "Failed to send to any peer".to_string(), - )); + { + let mut subscribers = subscribers.lock().await; + if let Some(list) = mt.and_then(|t| subscribers.get_mut(&t)) { + let last = list.len().saturating_sub(1); + let mut msg = Some(Arc::new(msg)); + let mut idx = 0; + + list.retain(|tx| { + // Hand the *last* subscriber our own reference instead of a + // clone. Every heavy message type (block, cfilter, cfheaders, + // headers) has exactly one subscriber, so it arrives with a + // refcount of 1 and the manager can take the payload without + // copying it. Only `inv`, which is tiny, is really shared. + let shared = if idx == last { + msg.take().expect("taken once, on the final subscriber") + } else { + Arc::clone( + msg.as_ref().expect("held until the final subscriber"), + ) + }; + idx += 1; + + tx.send((addr, shared)).is_ok() + }); + } + } + crate::timer::P_PUMP_BUSY.add(t_busy.elapsed()); + } + PeerEvent::Disconnected(addr) => { + let remaining = { + let mut guard = connected.lock().await; + guard.retain(|(peer, _)| peer.addr() != addr); + guard.len() + }; + tracing::info!( + target: "dash_spv::network", + "peer disconnected: {} | {} peers remaining", + addr, + remaining, + ); + let _ = events.send(NetworkEvent::PeerDisconnected(addr)); } - - Ok(()) } - } // end match - } // end send_message - - fn peer_count(&self) -> usize { - // Use cached counter to avoid blocking in async context - self.connected_peer_count.load(Ordering::Relaxed) - } - - async fn broadcast(&self, message: NetworkMessage) -> NetworkResult<()> { - let results = PeerNetworkManager::broadcast(self, message).await; - - if results.is_empty() { - return Err(NetworkError::ConnectionFailed("No connected peers".to_string())); } - - let successes = results.iter().filter(|r| r.is_ok()).count(); - if successes == 0 { - return Err(NetworkError::ConnectionFailed("All broadcast sends failed".to_string())); - } - Ok(()) - } - - async fn dispatch_local(&self, message: NetworkMessage) { - let local_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)); - let msg = Message::new(local_addr, message); - self.message_dispatcher.lock().await.dispatch(&msg); - } - - async fn disconnect_peer(&self, addr: &SocketAddr, reason: &str) -> NetworkResult<()> { - PeerNetworkManager::disconnect_peer(self, addr, reason) - .await - .map_err(|e| NetworkError::ConnectionFailed(e.to_string())) - } - - fn subscribe_network_events(&self) -> broadcast::Receiver { - self.network_event_sender.subscribe() - } + }) } -#[cfg(test)] -impl PeerNetworkManager { - pub(crate) async fn new_for_test(required_services: ServiceFlags) -> Self { - let test_dir = tempfile::tempdir().expect("test dir creation failed").keep(); - let peer_store = - PersistentPeerStorage::open(&test_dir).await.expect("test peer store init failed"); - let discovery = DnsDiscovery::new(); - let (request_tx, request_rx) = unbounded_channel(); +impl MsgQueue { + fn new() -> Self { Self { - pool: Arc::new(PeerPool::new(8)), - max_peers: 8, - discovery: Arc::new(discovery), - addrv2_handler: Arc::new(AddrV2Handler::new()), - peer_store: Arc::new(peer_store), - reputation_manager: Arc::new(PeerReputationManager::new()), - network: Network::Testnet, - shutdown_token: CancellationToken::new(), - tasks: Arc::new(Mutex::new(JoinSet::new())), - initial_peers: vec![], - data_dir: test_dir, - user_agent: None, - exclusive_mode: false, - required_services, - capability_rejected: Arc::new(RwLock::new(HashMap::new())), - connected_peer_count: Arc::new(AtomicUsize::new(0)), - headers2_disabled: Arc::new(Mutex::new(HashSet::new())), - message_dispatcher: Arc::new(Mutex::new(MessageDispatcher::default())), - request_tx, - request_rx: Arc::new(Mutex::new(Some(request_rx))), - round_robin_counter: Arc::new(AtomicUsize::new(0)), - network_event_sender: broadcast::Sender::new(DEFAULT_NETWORK_EVENT_CAPACITY), + other: Mutex::new(VecDeque::with_capacity(30)), + headers: Mutex::new(VecDeque::with_capacity(30)), + cfheaders: Mutex::new(VecDeque::with_capacity(30)), + cfilters: Mutex::new(VecDeque::with_capacity(30)), + blocks: Mutex::new(VecDeque::with_capacity(30)), + turn: AtomicUsize::new(0), + len: AtomicUsize::new(0), + notify: Notify::new(), } } - pub(crate) async fn insert_test_peer(&self, addr: SocketAddr, flags: ServiceFlags) { - self.pool.insert_peer_with_services(addr, flags).await; - self.connected_peer_count.fetch_add(1, Ordering::Relaxed); + fn len(&self) -> usize { + self.len.load(Ordering::SeqCst) } - pub(crate) async fn test_peer_count(&self) -> usize { - self.pool.peer_count().await + fn queue(&self, class: MsgClass) -> &Mutex> { + match class { + MsgClass::Other => &self.other, + MsgClass::Headers => &self.headers, + MsgClass::CfHeaders => &self.cfheaders, + MsgClass::CFilters => &self.cfilters, + MsgClass::Blocks => &self.blocks, + } } - pub(crate) async fn test_is_connected(&self, addr: &SocketAddr) -> bool { - self.pool.is_connected(addr).await - } + /// Take up to `n` messages in scheduling order: all pending control traffic + /// first (it is small, and something is usually blocked on it), then the bulk + /// classes by WEIGHTED rotation. + /// + /// Filters get most of the turns because they are the phase that actually costs + /// time: a compact filter is ~700 bytes per block against 32 for a filter + /// header, so the filter download is what the sync waits on. + /// + /// But not ALL the turns — strict priority would be a trap. Filters can only be + /// verified (and therefore stored, scanned and committed) once the filter-header + /// chain has been verified through their range, so starving cfheaders behind a + /// deep cfilters backlog stalls the very phase we were trying to favour. The + /// weights keep the small, cheap requests flowing while filters take the rest. + async fn pop_n(&self, n: usize) -> Vec { + if n == 0 { + return Vec::new(); + } + let mut out: Vec = Vec::with_capacity(n); - pub(crate) async fn insert_test_capability_rejected(&self, addr: SocketAddr) { - self.record_capability_rejection(addr).await; - } + { + let mut other = self.other.lock().await; + let take = n.min(other.len()); + out.extend(other.drain(..take)); + } - pub(crate) async fn test_capability_rejected_count(&self) -> usize { - self.capability_rejected.read().await.len() - } + // One lap of the rotation: filters four turns for every one of each header + // class. A class with an empty queue simply yields its turn. + // Filters take half the lap (they are the bytes), blocks a quarter (they + // gate the gap-limit cascade, so stalling them stretches the tail), and the + // two small header classes an eighth each — enough to keep the chains that + // unblock the filters moving. + const ROTATION: [MsgClass; 8] = [ + MsgClass::CFilters, + MsgClass::Blocks, + MsgClass::CFilters, + MsgClass::Headers, + MsgClass::CFilters, + MsgClass::Blocks, + MsgClass::CFilters, + MsgClass::CfHeaders, + ]; + let mut turn = self.turn.load(Ordering::Relaxed); + while out.len() < n { + let mut got = false; + for _ in 0..ROTATION.len() { + if out.len() == n { + break; + } + let class = ROTATION[turn % ROTATION.len()]; + turn = turn.wrapping_add(1); + if let Some(msg) = self.queue(class).lock().await.pop_front() { + out.push(msg); + got = true; + } + } + if !got { + break; // every bulk class is empty + } + } + self.turn.store(turn, Ordering::Relaxed); - pub(crate) async fn test_is_capability_rejected(&self, addr: &SocketAddr) -> bool { - self.is_capability_rejected(addr).await + self.len.fetch_sub(out.len(), Ordering::SeqCst); + out } - pub(crate) async fn test_has_capable_peer(&self) -> bool { - self.required_services != ServiceFlags::NONE - && self.pool.has_peers_with_service(self.required_services).await + async fn push(&self, msg: NetworkMessage) { + self.queue(classify(&msg)).lock().await.push_back(msg); + self.len.fetch_add(1, Ordering::SeqCst); + self.notify.notify_one(); } - pub(crate) async fn test_should_reject_after_handshake(&self, peer: &Peer) -> bool { - Self::should_reject_after_handshake(&self.pool, peer, self.required_services).await + /// Return messages that were popped but could not be sent to the FRONT of + /// their own class queue, preserving order. A popped message must never be + /// dropped: the owning pipeline has already recorded it as handed to the + /// network and only starts its response timeout once the router reports it on + /// the wire, so a dropped message is one the pipeline waits on forever — a + /// permanent sync stall (observed as: queue backed up, 0 MB/s, no sends, no + /// timeouts). + async fn push_front_all(&self, msgs: Vec) { + if msgs.is_empty() { + return; + } + let n = msgs.len(); + for msg in msgs.into_iter().rev() { + self.queue(classify(&msg)).lock().await.push_front(msg); + } + self.len.fetch_add(n, Ordering::SeqCst); + self.notify.notify_one(); } } diff --git a/dash-spv/src/network/message_dispatcher.rs b/dash-spv/src/network/message_dispatcher.rs deleted file mode 100644 index d88540d7d..000000000 --- a/dash-spv/src/network/message_dispatcher.rs +++ /dev/null @@ -1,227 +0,0 @@ -//! Message dispatcher for network message distribution. -//! -//! This module filters incoming network messages by type and forwards -//! them to registered receivers. -//! -//! - [`Message`]: Wraps a `NetworkMessage` with the originating peer address -//! - [`MessageDispatcher`]: Manages channels and dispatches messages to interested parties - -use std::collections::{HashMap, HashSet}; -use std::net::SocketAddr; - -use dashcore::network::message::NetworkMessage; -use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; - -use crate::network::MessageType; - -/// A network message tagged with the peer that sent it. -#[derive(Clone, Debug, PartialEq)] -pub struct Message { - peer_address: SocketAddr, - inner: NetworkMessage, -} - -impl Message { - /// Creates a new Message from a peer address and network message. - pub fn new(peer_address: SocketAddr, inner: NetworkMessage) -> Self { - Self { - peer_address, - inner, - } - } - - /// Forwards the cmd() of the underlying NetworkMessage. - pub fn cmd(&self) -> &'static str { - self.inner.cmd() - } - - /// Returns the SocketAddr of the peer that sent this message. - pub fn peer_address(&self) -> SocketAddr { - self.peer_address - } - - /// Returns a reference to the underlying network message. - pub fn inner(&self) -> &NetworkMessage { - &self.inner - } -} - -/// Routes incoming network messages to subscribers based on message type. -/// -/// Subscribers call [`message_receiver`](Self::message_receiver) with the message types they -/// want, receiving an unbounded channel. When [`dispatch`](Self::dispatch) is called, the -/// message is sent to all subscribers registered for that type. Dead channels are pruned -/// automatically on dispatch. -#[derive(Debug, Default)] -pub struct MessageDispatcher { - senders: HashMap>>, -} - -impl MessageDispatcher { - /// Creates and returns a receiver that yields only messages matching the provided message types. - pub fn message_receiver( - &mut self, - message_types: &[MessageType], - ) -> UnboundedReceiver { - let (sender, receiver) = unbounded_channel(); - let unique_types: HashSet = message_types.iter().copied().collect(); - for message_type in unique_types { - self.senders.entry(message_type).or_default().push(sender.clone()); - } - receiver - } - - /// Distributes a message to all subscribers interested in its type. Prunes dead senders automatically. - pub fn dispatch(&mut self, message: &Message) { - let message_type = MessageType::from(message); - if let Some(senders) = self.senders.get_mut(&message_type) { - senders.retain(|sender| sender.send(message.clone()).is_ok()); - }; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::test_socket_address; - - #[test] - fn test_message_creation() { - let peer_address = test_socket_address(1); - let inner = NetworkMessage::Headers(vec![]); - let msg = Message::new(peer_address, inner.clone()); - - assert_eq!(msg.cmd(), inner.cmd()); - assert_eq!(msg.peer_address(), peer_address); - assert_eq!(*msg.inner(), inner); - } - - #[tokio::test] - async fn test_dispatch_to_interested_receiver() { - let mut message_dispatcher = MessageDispatcher::default(); - let mut receiver = message_dispatcher.message_receiver(&[MessageType::Inv]); - - let msg = Message::new(test_socket_address(1), NetworkMessage::Inv(vec![])); - message_dispatcher.dispatch(&msg); - - assert_eq!(receiver.recv().await.unwrap(), msg); - } - - #[tokio::test] - async fn test_dispatch_skips_uninterested() { - let mut message_dispatcher = MessageDispatcher::default(); - let mut receiver = message_dispatcher.message_receiver(&[MessageType::Headers]); - - let msg = Message::new(test_socket_address(1), NetworkMessage::Inv(vec![])); - message_dispatcher.dispatch(&msg); - - assert!(receiver.try_recv().is_err()); - } - - #[test] - fn test_dispatch_no_subscribers() { - let mut message_dispatcher = MessageDispatcher::default(); - let msg = Message::new(test_socket_address(1), NetworkMessage::Headers(vec![])); - - // Should not panic with no subscribers - message_dispatcher.dispatch(&msg); - } - - #[tokio::test] - async fn test_message_receiver_multiple_types() { - let mut message_dispatcher = MessageDispatcher::default(); - let mut receiver1 = message_dispatcher.message_receiver(&[MessageType::Headers]); - let mut receiver2 = message_dispatcher.message_receiver(&[MessageType::Inv]); - - let headers_msg = Message::new(test_socket_address(1), NetworkMessage::Headers(vec![])); - let inv_msg = Message::new(test_socket_address(2), NetworkMessage::Inv(vec![])); - - message_dispatcher.dispatch(&headers_msg); - message_dispatcher.dispatch(&inv_msg); - - assert_eq!(receiver1.recv().await.unwrap(), headers_msg); - assert_eq!(receiver2.recv().await.unwrap(), inv_msg); - } - - #[tokio::test] - async fn test_dispatch_multiple_subscribers_same_type() { - let mut message_dispatcher = MessageDispatcher::default(); - let mut receiver1 = message_dispatcher.message_receiver(&[MessageType::Headers]); - let mut receiver2 = message_dispatcher.message_receiver(&[MessageType::Headers]); - - let msg = Message::new(test_socket_address(1), NetworkMessage::Headers(vec![])); - message_dispatcher.dispatch(&msg); - - assert_eq!(receiver1.recv().await.unwrap(), msg); - assert_eq!(receiver2.recv().await.unwrap(), msg); - } - - #[tokio::test] - async fn test_dropped_receiver_does_not_affect_others() { - let mut message_dispatcher = MessageDispatcher::default(); - let receiver1 = message_dispatcher.message_receiver(&[MessageType::Headers]); - let mut receiver2 = message_dispatcher.message_receiver(&[MessageType::Headers]); - - drop(receiver1); - - let msg = Message::new(test_socket_address(1), NetworkMessage::Headers(vec![])); - message_dispatcher.dispatch(&msg); - - assert_eq!(receiver2.recv().await.unwrap(), msg); - } - - #[tokio::test] - async fn test_messages_received_in_order() { - let mut message_dispatcher = MessageDispatcher::default(); - let mut receiver = - message_dispatcher.message_receiver(&[MessageType::Headers, MessageType::Inv]); - - let peer_1 = test_socket_address(1); - let peer_2 = test_socket_address(2); - let peer_3 = test_socket_address(3); - - let msg1 = Message::new(peer_1, NetworkMessage::Headers(vec![])); - let msg2 = Message::new(peer_2, NetworkMessage::Inv(vec![])); - let msg3 = Message::new(peer_3, NetworkMessage::Headers(vec![])); - - message_dispatcher.dispatch(&msg1); - message_dispatcher.dispatch(&msg2); - message_dispatcher.dispatch(&msg3); - - assert_eq!(receiver.recv().await.unwrap().peer_address(), peer_1); - assert_eq!(receiver.recv().await.unwrap().peer_address(), peer_2); - assert_eq!(receiver.recv().await.unwrap().peer_address(), peer_3); - } - - #[tokio::test] - async fn test_message_receiver_receives_multiple_types() { - let mut message_dispatcher = MessageDispatcher::default(); - let mut receiver = - message_dispatcher.message_receiver(&[MessageType::Headers, MessageType::Inv]); - - let headers_msg = Message::new(test_socket_address(1), NetworkMessage::Headers(vec![])); - let inv_msg = Message::new(test_socket_address(2), NetworkMessage::Inv(vec![])); - - message_dispatcher.dispatch(&headers_msg); - message_dispatcher.dispatch(&inv_msg); - - assert_eq!(receiver.recv().await.unwrap(), headers_msg); - assert_eq!(receiver.recv().await.unwrap(), inv_msg); - } - - #[tokio::test] - async fn test_duplicate_message_types_no_duplicate_delivery() { - let mut message_dispatcher = MessageDispatcher::default(); - let mut receiver = message_dispatcher.message_receiver(&[ - MessageType::Headers, - MessageType::Headers, - MessageType::Headers, - ]); - - let msg = Message::new(test_socket_address(1), NetworkMessage::Headers(vec![])); - message_dispatcher.dispatch(&msg); - - assert_eq!(receiver.recv().await.unwrap(), msg); - assert!(receiver.try_recv().is_err()); - } -} diff --git a/dash-spv/src/network/message_type.rs b/dash-spv/src/network/message_type.rs deleted file mode 100644 index 9d2dfb86d..000000000 --- a/dash-spv/src/network/message_type.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! Message type enum for easier message mapping to NetworkMessage variants. -//! -//! Uses a macro to keep MessageType in sync with NetworkMessage. -//! If NetworkMessage adds a new variant, compilation will fail until -//! the variant is added here. - -use crate::network::Message; -use dashcore::network::message::NetworkMessage; - -/// Generates the `MessageType` enum -/// -/// Implements: -/// - `From<&Message>` -/// -/// Each `NetworkMessage` variant maps to a corresponding `MessageType` variant -/// (e.g., `NetworkMessage::Headers(_)` -> `MessageType::Headers`). -/// -/// Syntax for entries: -/// - `Name` for unit variants (e.g., `Verack`) -/// - `Name (..)` for tuple variants with data (e.g., `Headers (..)`) -/// - `Name { .. }` for struct variants (e.g., `Unknown { .. }`) -macro_rules! define_message_types { - ($($(#[$meta:meta])* $variant:ident $( ( $($tuple:tt)* ) )? $( { $($field:tt)* } )?),* $(,)?) => { - /// Message types that subscribers can subscribe to. - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] - pub enum MessageType { - $($(#[$meta])* $variant,)* - } - - impl From<&Message> for MessageType { - fn from(value: &Message) -> Self { - match value.inner() { - $(NetworkMessage::$variant $( ( $($tuple)* ) )? $( { $($field)* } )? => MessageType::$variant,)* - } - } - } - }; -} - -define_message_types! { - /// `version` - Version (..), - /// `verack` - Verack, - /// `addr` - Addr (..), - /// `inv` - Inv (..), - /// `getdata` - GetData (..), - /// `notfound` - NotFound (..), - /// `getblocks` - GetBlocks (..), - /// `getheaders` - GetHeaders (..), - /// `mempool` - MemPool, - /// `tx` - Tx (..), - /// `block` - Block (..), - /// `headers` - Headers (..), - /// `sendheaders` - SendHeaders, - /// `getheaders2` - GetHeaders2 (..), - /// `sendheaders2` - SendHeaders2, - /// `headers2` - Headers2 (..), - /// `getaddr` - GetAddr, - /// `ping` - Ping (..), - /// `pong` - Pong (..), - /// `merkleblock` - MerkleBlock (..), - /// `filterload` - FilterLoad (..), - /// `filteradd` - FilterAdd (..), - /// `filterclear` - FilterClear, - /// `getcfilters` - GetCFilters (..), - /// `cfilter` - CFilter (..), - /// `getcfheaders` - GetCFHeaders (..), - /// `cfheaders` - CFHeaders (..), - /// `getcfcheckpt` - GetCFCheckpt (..), - /// `cfcheckpt` - CFCheckpt (..), - /// `sendcmpct` - SendCmpct (..), - /// `cmpctblock` - CmpctBlock (..), - /// `getblocktxn` - GetBlockTxn (..), - /// `blocktxn` - BlockTxn (..), - /// `alert` - Alert (..), - /// `reject` - Reject (..), - /// `feefilter` - FeeFilter (..), - /// `wtxidrelay` - WtxidRelay, - /// `addrv2` - AddrV2 (..), - /// `sendaddrv2` - SendAddrV2, - /// `getmnlistd` - GetMnListD (..), - /// `mnlistdiff` - MnListDiff (..), - /// `getqrinfo` - GetQRInfo (..), - /// `qrinfo` - QRInfo (..), - /// `clsig` - CLSig (..), - /// `isdlock` - ISLock (..), - /// `senddsq` - SendDsq (..), - /// Unknown message type - Unknown { .. }, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::test_socket_address; - - #[test] - fn from_message_unit_variant() { - let addr = test_socket_address(1); - - let msg = Message::new(addr, NetworkMessage::SendHeaders); - assert_eq!(MessageType::from(&msg), MessageType::SendHeaders); - } - - #[test] - fn from_message_tuple_variant() { - let addr = test_socket_address(1); - - let msg = Message::new(addr, NetworkMessage::Alert(vec![])); - assert_eq!(MessageType::from(&msg), MessageType::Alert); - } - - #[test] - fn from_message_unknown_variant() { - use dashcore::network::message::CommandString; - - let addr = test_socket_address(1); - let unknown_msg = NetworkMessage::Unknown { - command: CommandString::try_from_static("test").unwrap(), - payload: vec![], - }; - let msg = Message::new(addr, unknown_msg); - assert_eq!(MessageType::from(&msg), MessageType::Unknown); - } -} diff --git a/dash-spv/src/network/mod.rs b/dash-spv/src/network/mod.rs index 017af47d9..49b4e53fe 100644 --- a/dash-spv/src/network/mod.rs +++ b/dash-spv/src/network/mod.rs @@ -1,263 +1,10 @@ -//! Network layer for the Dash SPV client. - -pub mod addrv2; -pub mod constants; -pub mod discovery; -mod event; -pub mod handshake; -pub mod manager; -mod message_dispatcher; -pub mod peer; -pub mod pool; -mod reputation; - -mod message_type; -#[cfg(test)] -mod tests; - -pub use event::NetworkEvent; - -use async_trait::async_trait; -use tokio::sync::{broadcast, mpsc}; - -use crate::error::NetworkResult; -use crate::NetworkError; -use dashcore::network::message::NetworkMessage; -use dashcore::network::message_blockdata::{GetHeadersMessage, Inventory}; -use dashcore::network::message_bloom::FilterLoad; -use dashcore::network::message_filter::{GetCFHeaders, GetCFilters}; -use dashcore::network::message_qrinfo::GetQRInfo; -use dashcore::network::message_sml::GetMnListDiff; -use dashcore::BlockHash; -use dashcore_hashes::Hash; -pub use handshake::{HandshakeManager, HandshakeState}; -pub use manager::PeerNetworkManager; -pub use message_dispatcher::{Message, MessageDispatcher}; -pub use message_type::MessageType; -pub use peer::Peer; -pub(crate) use reputation::PeerReputation; -use std::net::SocketAddr; -use tokio::sync::mpsc::UnboundedReceiver; - -const FILTER_TYPE_DEFAULT: u8 = 0; - -/// Request to send to network. -#[derive(Debug)] -pub enum NetworkRequest { - /// Send a message to the network. - SendMessage(NetworkMessage), - /// Send a message to a specific peer. - SendMessageToPeer(NetworkMessage, SocketAddr), - /// Broadcast a message to all connected peers. - BroadcastMessage(NetworkMessage), -} - -/// Handle for managers to queue outgoing network requests. -#[derive(Clone)] -pub struct RequestSender { - tx: mpsc::UnboundedSender, -} - -impl RequestSender { - /// Create a new RequestSender. - pub fn new(tx: mpsc::UnboundedSender) -> Self { - Self { - tx, - } - } - - /// Queue a message to be sent to the network. - fn send_message(&self, msg: NetworkMessage) -> NetworkResult<()> { - self.tx - .send(NetworkRequest::SendMessage(msg)) - .map_err(|e| NetworkError::ProtocolError(e.to_string())) - } - - /// Queue a message to be sent to a specific peer. - fn send_message_to_peer( - &self, - msg: NetworkMessage, - peer_address: SocketAddr, - ) -> NetworkResult<()> { - self.tx - .send(NetworkRequest::SendMessageToPeer(msg, peer_address)) - .map_err(|e| NetworkError::ProtocolError(e.to_string())) - } - - /// Queue a message to be broadcast to all connected peers. - pub(crate) fn broadcast(&self, msg: NetworkMessage) -> NetworkResult<()> { - self.tx - .send(NetworkRequest::BroadcastMessage(msg)) - .map_err(|e| NetworkError::ProtocolError(e.to_string())) - } - - /// Request inventory from a specific peer. - pub fn request_inventory( - &self, - inventory: Vec, - peer_address: SocketAddr, - ) -> NetworkResult<()> { - self.send_message_to_peer(NetworkMessage::GetData(inventory), peer_address) - } - - pub fn request_block_headers(&self, start_hash: BlockHash) -> NetworkResult<()> { - self.send_message(NetworkMessage::GetHeaders(GetHeadersMessage::new( - vec![start_hash], - BlockHash::all_zeros(), - ))) - } - - pub fn request_block_headers_from_peer( - &self, - start_hash: BlockHash, - address: SocketAddr, - ) -> NetworkResult<()> { - self.send_message_to_peer( - NetworkMessage::GetHeaders(GetHeadersMessage::new( - vec![start_hash], - BlockHash::all_zeros(), - )), - address, - ) - } - - pub fn request_filter_headers( - &self, - start_height: u32, - stop_hash: BlockHash, - ) -> NetworkResult<()> { - self.send_message(NetworkMessage::GetCFHeaders(GetCFHeaders { - filter_type: FILTER_TYPE_DEFAULT, - start_height, - stop_hash, - })) - } - - pub fn request_filters(&self, start_height: u32, stop_hash: BlockHash) -> NetworkResult<()> { - self.send_message(NetworkMessage::GetCFilters(GetCFilters { - filter_type: FILTER_TYPE_DEFAULT, - start_height, - stop_hash, - })) - } - - pub fn request_mnlist_diff( - &self, - base_block_hash: BlockHash, - block_hash: BlockHash, - ) -> NetworkResult<()> { - self.send_message(NetworkMessage::GetMnListD(GetMnListDiff { - base_block_hash, - block_hash, - })) - } - - pub fn request_qr_info( - &self, - known_block_hashes: Vec, - target_block_hash: BlockHash, - extra_share: bool, - ) -> NetworkResult<()> { - self.send_message(NetworkMessage::GetQRInfo(GetQRInfo { - base_block_hashes: known_block_hashes, - block_request_hash: target_block_hash, - extra_share, - })) - } - - pub fn request_blocks(&self, hashes: Vec) -> NetworkResult<()> { - self.send_message(NetworkMessage::GetData( - hashes.into_iter().map(Inventory::Block).collect(), - )) - } - - /// Send a filterload message to a specific peer. - pub fn send_filter_load(&self, filter_load: FilterLoad, peer: SocketAddr) -> NetworkResult<()> { - self.send_message_to_peer(NetworkMessage::FilterLoad(filter_load), peer) - } - - /// Send a filterclear message to a specific peer. - pub fn send_filter_clear(&self, peer: SocketAddr) -> NetworkResult<()> { - self.send_message_to_peer(NetworkMessage::FilterClear, peer) - } - - /// Send a mempool message to request inventory from a specific peer. - pub fn request_mempool(&self, peer: SocketAddr) -> NetworkResult<()> { - self.send_message_to_peer(NetworkMessage::MemPool, peer) - } -} - -/// Network manager trait for abstracting network operations. -#[async_trait] -pub trait NetworkManager: Send + Sync + 'static { - /// Creates and returns a receiver that yields only messages of the matching the provided message types. - async fn message_receiver(&mut self, types: &[MessageType]) -> UnboundedReceiver; - - /// Get a sender for queuing outgoing network requests. - /// - /// Messages sent via this sender are delivered to the network asynchronously. - fn request_sender(&self) -> RequestSender; - - /// Connect to the network. - async fn connect(&mut self) -> NetworkResult<()>; - - /// Disconnect from the network. - async fn disconnect(&mut self) -> NetworkResult<()>; - - /// Send a message to a peer. - async fn send_message(&mut self, message: NetworkMessage) -> NetworkResult<()>; - - /// Get the number of connected peers. - fn peer_count(&self) -> usize; - - /// Request QRInfo from the network. - /// - /// # Arguments - /// * `base_block_hashes` - Array of base block hashes for the masternode lists the light client already knows - /// * `block_request_hash` - Hash of the block for which the masternode list diff is requested - /// * `extra_share` - Optional flag to indicate if an extra share is requested - async fn request_qr_info( - &mut self, - base_block_hashes: Vec, - block_request_hash: BlockHash, - extra_share: bool, - ) -> NetworkResult<()> { - use dashcore::network::message_qrinfo::GetQRInfo; - - let get_qr_info = GetQRInfo { - base_block_hashes: base_block_hashes.clone(), - block_request_hash, - extra_share, - }; - - let base_hashes_count = get_qr_info.base_block_hashes.len(); - - self.send_message(NetworkMessage::GetQRInfo(get_qr_info)).await?; - - tracing::debug!( - "Requested QRInfo with {} base hashes for block {}, extra_share={}", - base_hashes_count, - block_request_hash, - extra_share - ); - - Ok(()) - } - - /// Broadcast a message to all connected peers. - async fn broadcast(&self, _message: NetworkMessage) -> NetworkResult<()>; - - /// Inject a message into the local message dispatcher as if received from a peer. - /// - /// Used for locally-originated messages (e.g., self-broadcast transactions) that - /// should be processed through the same pipeline as peer-received messages. - async fn dispatch_local(&self, message: NetworkMessage); - - /// Disconnect a specific peer by address. - async fn disconnect_peer(&self, _addr: &SocketAddr, _reason: &str) -> NetworkResult<()>; - - /// Subscribe to network events (peer connections, disconnections). - /// - /// Returns a broadcast receiver for network events. - fn subscribe_network_events(&self) -> broadcast::Receiver; -} +mod discovery; +mod manager; +mod peer; + +// `NetworkEvent` is part of the public `EventHandler` API (delivered to +// `on_network_event`), so it stays exported. The peer-to-peer manager and its +// request/message plumbing are internal: the client builds and owns the manager +// itself (see `DashSpvClient::new`), so none of these are part of the public API. +pub use manager::NetworkEvent; +pub(crate) use manager::{MessageType, PeerNetworkManager, RequestKey}; diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 2966ac52d..4511d4ff9 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -1,850 +1,542 @@ -//! Dash peer connection management. - -use dashcore::network::constants::ServiceFlags; -use std::collections::HashMap; +use std::collections::VecDeque; use std::net::SocketAddr; +use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; -use std::time::{Duration, SystemTime}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpStream; -use tokio::sync::Mutex; - -use dashcore::consensus::{encode, Decodable}; -use dashcore::network::message::{NetworkMessage, RawNetworkMessage}; -use dashcore::Network; - -use crate::error::{NetworkError, NetworkResult}; -use crate::network::constants::PING_INTERVAL; -use crate::network::Message; - -/// Internal state for the TCP connection -struct ConnectionState { - stream: TcpStream, - // Stateful message framing buffer to ensure full frames before decoding - framing_buffer: Vec, -} - -/// Dash P2P peer -pub struct Peer { - address: SocketAddr, - // Use a single mutex to protect both the write stream and read buffer - // This ensures no concurrent access to the underlying socket - state: Option>>, - timeout: Duration, - connected_at: Option, - bytes_sent: u64, - network: Network, - // Ping/pong state - last_ping_sent: Option, - last_pong_received: Option, - pending_pings: HashMap, // nonce -> sent_time - // Peer information from Version message - version: Option, - services: Option, - user_agent: Option, - best_height: Option, - relay: Option, - prefers_headers2: bool, - sent_sendheaders2: bool, +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +/// Per-peer response-latency tracker: the time each pipeline request spends in +/// flight, from send to the response that completes it. Send times are queued +/// FIFO; each completing response pops the oldest and folds the elapsed time +/// into running total/count/max (so we can report avg and worst per peer). +#[derive(Default)] +pub(crate) struct Latency { + pending: Mutex>, + total_ns: AtomicU64, + count: AtomicU64, + max_ns: AtomicU64, } -impl Peer { - /// Get the remote peer socket address. - pub fn address(&self) -> SocketAddr { - self.address - } - /// Create a new peer. - pub fn new(address: SocketAddr, timeout: Duration, network: Network) -> Self { - Self { - address, - state: None, - timeout, - connected_at: None, - bytes_sent: 0, - network, - last_ping_sent: None, - last_pong_received: None, - pending_pings: HashMap::new(), - version: None, - services: None, - user_agent: None, - best_height: None, - relay: None, - prefers_headers2: false, - sent_sendheaders2: false, - } +impl Latency { + async fn on_send(&self) { + self.pending.lock().await.push_back(Instant::now()); } - /// Connect to a peer and return a connected instance. - pub async fn connect( - address: SocketAddr, - timeout_secs: u64, - network: Network, - ) -> NetworkResult { - let timeout = Duration::from_secs(timeout_secs); - - let stream = tokio::time::timeout(timeout, TcpStream::connect(address)) - .await - .map_err(|_| { - NetworkError::ConnectionFailed(format!("Connection to {} timed out", address)) - })? - .map_err(|e| { - NetworkError::ConnectionFailed(format!("Failed to connect to {}: {}", address, e)) - })?; - - stream.set_nodelay(true).map_err(|e| { - NetworkError::ConnectionFailed(format!("Failed to set TCP_NODELAY: {}", e)) - })?; - - let state = ConnectionState { - stream, - framing_buffer: Vec::new(), - }; - - Ok(Self { - address, - state: Some(Arc::new(Mutex::new(state))), - timeout, - connected_at: Some(SystemTime::now()), - bytes_sent: 0, - network, - last_ping_sent: None, - last_pong_received: None, - pending_pings: HashMap::new(), - version: None, - services: None, - user_agent: None, - best_height: None, - relay: None, - prefers_headers2: false, - sent_sendheaders2: false, - }) - } - - pub fn version(&self) -> Option { - self.version + /// Drop pending sends older than `timeout` (the pipelines have long since + /// timed them out and re-queued them elsewhere), returning how many. Their + /// in-flight slots would otherwise be held forever — see + /// [`ConnectedPeer::reap_stale`]. + async fn reap(&self, timeout: Duration) -> usize { + let mut pending = self.pending.lock().await; + let now = Instant::now(); + let mut n = 0; + while let Some(&front) = pending.front() { + if now.duration_since(front) > timeout { + pending.pop_front(); + n += 1; + } else { + break; // FIFO: the rest are younger + } + } + n } - pub fn best_height(&self) -> Option { - self.best_height + /// Pop the oldest pending send and record its round-trip. + async fn complete_one(&self) { + let sent = self.pending.lock().await.pop_front(); + if let Some(sent) = sent { + let ns = sent.elapsed().as_nanos() as u64; + self.total_ns.fetch_add(ns, Ordering::Relaxed); + self.count.fetch_add(1, Ordering::Relaxed); + self.max_ns.fetch_max(ns, Ordering::Relaxed); + } } - /// Check if peer supports compact filters (BIP 157/158). - pub fn supports_compact_filters(&self) -> bool { - self.has_service(ServiceFlags::COMPACT_FILTERS) + /// Cumulative (completed request count, total service-time nanoseconds). + /// Deltas between two reads give the windowed completion rate and average + /// service time used to size the in-flight budget by Little's Law. + fn totals(&self) -> (u64, u64) { + (self.count.load(Ordering::Relaxed), self.total_ns.load(Ordering::Relaxed)) } - /// Check if peer supports headers2 compression (DIP-0025). - pub fn supports_headers2(&self) -> bool { - self.has_service(ServiceFlags::NODE_HEADERS_COMPRESSED) + /// (completed request count, average ms, worst ms). + fn snapshot(&self) -> (u64, f64, f64) { + let count = self.count.load(Ordering::Relaxed); + let total = self.total_ns.load(Ordering::Relaxed); + let max = self.max_ns.load(Ordering::Relaxed); + let avg_ms = if count > 0 { + total as f64 / count as f64 / 1e6 + } else { + 0.0 + }; + (count, avg_ms, max as f64 / 1e6) } +} - pub fn has_service(&self, flags: ServiceFlags) -> bool { - self.services.map(|s| ServiceFlags::from(s).has(flags)).unwrap_or(false) - } +use dashcore::{ + consensus::encode, + network::{ + address::Address, + constants::ServiceFlags, + message::{NetworkMessage, RawNetworkMessage, RawNetworkMessageCodec}, + message_network::VersionMessage, + }, + Network, +}; +use futures::lock::Mutex; +use tokio::sync::mpsc::UnboundedSender; +use tokio::{ + io::{AsyncRead, AsyncWriteExt, ReadBuf}, + net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, + }, +}; +use tokio_stream::StreamExt; +use tokio_util::codec::FramedRead; +use tokio_util::sync::CancellationToken; + +use crate::{error::NetworkResult, NetworkError}; + +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); +const USER_AGENT: &str = concat!("/dash-spv:", env!("CARGO_PKG_VERSION"), "/"); + +/// Wraps a socket read half and adds every byte read into a shared counter, so +/// the network manager can estimate download throughput (and size the global +/// in-flight budget to ~90% of it). +struct CountingReader { + inner: R, + bytes: Arc, +} - pub(crate) fn services_known(&self) -> bool { - self.services.is_some() +impl AsyncRead for CountingReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> std::task::Poll> { + let before = buf.filled().len(); + let r = std::pin::Pin::new(&mut self.inner).poll_read(cx, buf); + if let std::task::Poll::Ready(Ok(())) = &r { + let n = buf.filled().len() - before; + self.bytes.fetch_add(n as u64, Ordering::Relaxed); + } + r } +} - /// Connect to the peer (instance method for compatibility). - pub async fn connect_instance(&mut self) -> NetworkResult<()> { - let stream = tokio::time::timeout(self.timeout, TcpStream::connect(self.address)) - .await - .map_err(|_| { - NetworkError::ConnectionFailed(format!("Connection to {} timed out", self.address)) - })? - .map_err(|e| { - NetworkError::ConnectionFailed(format!( - "Failed to connect to {}: {}", - self.address, e - )) - })?; - - // Disable Nagle's algorithm for lower latency - stream.set_nodelay(true).map_err(|e| { - NetworkError::ConnectionFailed(format!("Failed to set TCP_NODELAY: {}", e)) - })?; +type PeerReader = FramedRead, RawNetworkMessageCodec>; - let state = ConnectionState { - stream, - framing_buffer: Vec::new(), - }; +pub enum PeerEvent { + Message(SocketAddr, NetworkMessage), + Disconnected(SocketAddr), +} - self.state = Some(Arc::new(Mutex::new(state))); - self.connected_at = Some(SystemTime::now()); +pub struct ConnectedPeer { + network: Network, + addr: SocketAddr, + version: VersionMessage, + lag_ms: AtomicU32, + in_flight: Arc, + latency: Arc, + writer: Arc>, + /// This peer's measured serving capacity — the number of requests it can have + /// in flight before ITS responses start queuing (its bandwidth-delay product). + /// Sized continuously by the bandwidth controller from the peer's own + /// completion rate and service time, mirroring how the global budget is sized + /// from our download rate. Fast peers earn a high cap, slow peers a low one. + cap: Arc, + /// Per-connection cancel token (child of the global shutdown). Cancelling it + /// stops this peer's reader and closes the socket. Used to drop peers we + /// probed but don't keep, so we only hold connections we actually use. + token: CancellationToken, +} - tracing::info!("Connected to peer {}", self.address); +pub struct DisconnectedPeer { + network: Network, + addr: SocketAddr, +} - Ok(()) +impl ConnectedPeer { + pub fn addr(&self) -> SocketAddr { + self.addr } - /// Disconnect from the peer. - pub async fn disconnect(&mut self) -> NetworkResult<()> { - if let Some(state_arc) = self.state.take() { - if let Ok(state_mutex) = Arc::try_unwrap(state_arc) { - let mut state = state_mutex.into_inner(); - let _ = state.stream.shutdown().await; - } - } - self.connected_at = None; - - tracing::info!("Disconnected from peer {}", self.address); - - Ok(()) + pub fn version(&self) -> &VersionMessage { + &self.version } - /// Update peer information from a received Version message - pub fn update_peer_info( - &mut self, - version_msg: &dashcore::network::message_network::VersionMessage, - ) { - // Define validation constants - const MIN_PROTOCOL_VERSION: u32 = 60001; // Minimum version that supports ping/pong - const MAX_PROTOCOL_VERSION: u32 = 100000; // Reasonable upper bound for protocol version - const MAX_USER_AGENT_LENGTH: usize = 256; // Maximum reasonable user agent length - const MAX_START_HEIGHT: i32 = 10_000_000; // Reasonable upper bound for block height - - // Validate protocol version - if version_msg.version < MIN_PROTOCOL_VERSION { - tracing::warn!( - "Peer {} reported protocol version {} below minimum {}, skipping update", - self.address, - version_msg.version, - MIN_PROTOCOL_VERSION - ); - return; - } - - if version_msg.version > MAX_PROTOCOL_VERSION { - tracing::warn!( - "Peer {} reported suspiciously high protocol version {}, skipping update", - self.address, - version_msg.version - ); - return; - } - - // Validate start height - if version_msg.start_height < 0 { - tracing::warn!( - "Peer {} reported negative start height {}, skipping update", - self.address, - version_msg.start_height - ); - return; - } - - if version_msg.start_height > MAX_START_HEIGHT { - tracing::warn!( - "Peer {} reported suspiciously high start height {}, skipping update", - self.address, - version_msg.start_height - ); - return; - } - - // Validate user agent - if version_msg.user_agent.is_empty() { - tracing::warn!("Peer {} provided empty user agent, skipping update", self.address); - return; - } - - if version_msg.user_agent.len() > MAX_USER_AGENT_LENGTH { - tracing::warn!( - "Peer {} provided excessively long user agent ({} bytes), skipping update", - self.address, - version_msg.user_agent.len() - ); - return; - } - - // Validate services - ensure they contain expected flags - let services = version_msg.services.as_u64(); - const KNOWN_SERVICE_FLAGS: u64 = 0x0000_0000_0000_1FFF; // All known service flags up to bit 12 - if services & !KNOWN_SERVICE_FLAGS != 0 { - tracing::warn!( - "Peer {} reported unknown service flags: 0x{:016x}, proceeding with caution", - self.address, - services - ); - // Note: We don't return here as unknown flags might be from newer versions - } - - // All validations passed, update peer info - self.version = Some(version_msg.version); - self.services = Some(version_msg.services.as_u64()); - self.user_agent = Some(version_msg.user_agent.clone()); - self.best_height = Some(version_msg.start_height as u32); - self.relay = Some(version_msg.relay); - - tracing::info!( - "Updated peer info for {}: height={}, version={}, services={:?}", - self.address, - version_msg.start_height, - version_msg.version, - version_msg.services - ); - - // Also log with standard logging for debugging - tracing::info!( - "PEER_INFO_DEBUG: Updated peer {} with height={}, version={}", - self.address, - version_msg.start_height, - version_msg.version - ); + /// Net in-flight to this peer: `+1` per message we send it, `-1` per message + /// we read from it (managed internally by `send` and the reader task). The + /// router reads this to send to the least-loaded peer. + pub fn in_flight(&self) -> usize { + self.in_flight.load(Ordering::Relaxed) } - /// Helper function to read some bytes into the framing buffer. - async fn read_some(state: &mut ConnectionState) -> std::io::Result { - let mut tmp = [0u8; 8192]; - match state.stream.read(&mut tmp).await { - Ok(0) => Ok(0), - Ok(n) => { - state.framing_buffer.extend_from_slice(&tmp[..n]); - Ok(n) - } - Err(e) => Err(e), + pub fn disconnect(self) -> DisconnectedPeer { + DisconnectedPeer { + network: self.network, + addr: self.addr, } } - /// Send a message to the peer. - pub async fn send_message(&mut self, message: NetworkMessage) -> NetworkResult<()> { - let state_arc = self - .state - .as_ref() - .ok_or_else(|| NetworkError::ConnectionFailed("Not connected".to_string()))?; - - let raw_message = RawNetworkMessage { + pub async fn send(&self, msg: &NetworkMessage) -> NetworkResult<()> { + // TODO: Take a reference to msg instead of cloning it + let raw = RawNetworkMessage { magic: self.network.magic(), - payload: message, + payload: msg.clone(), }; + let serialized = encode::serialize(&raw); - let serialized = encode::serialize(&raw_message); - - // Log details for debugging headers2 issues - if matches!( - raw_message.payload, - NetworkMessage::GetHeaders2(_) | NetworkMessage::GetHeaders(_) - ) { - let msg_type = match raw_message.payload { - NetworkMessage::GetHeaders2(_) => "GetHeaders2", - NetworkMessage::GetHeaders(_) => "GetHeaders", - _ => "Unknown", - }; - tracing::debug!( - "Sending {} raw bytes (len={}): {:02x?}", - msg_type, - serialized.len(), - &serialized[..std::cmp::min(100, serialized.len())] - ); + if let Err(e) = self.writer.lock().await.write_all(&serialized).await { + tracing::warn!("Disconnecting {} due to write error: {}", self.addr, e); + return Err(NetworkError::ConnectionFailed(format!("Write failed: {}", e))); } - - // Lock the state for the entire write operation - let mut state = state_arc.lock().await; - - // Write with error handling - match state.stream.write_all(&serialized).await { - Ok(_) => { - // Flush to ensure data is sent immediately - if let Err(e) = state.stream.flush().await { - tracing::warn!("Failed to flush socket {}: {}", self.address, e); - } - self.bytes_sent += serialized.len() as u64; - tracing::debug!("Sent message to {}: {:?}", self.address, raw_message.payload); - Ok(()) - } - Err(e) => { - tracing::warn!("Disconnecting {} due to write error: {}", self.address, e); - // Drop the lock before clearing connection state - drop(state); - // Clear connection state on write error - self.state = None; - self.connected_at = None; - Err(NetworkError::ConnectionFailed(format!("Write failed: {}", e))) - } + // A pipeline request counts as one unit of in-flight work for this peer. + if is_pipeline_request(msg) { + self.in_flight.fetch_add(1, Ordering::Relaxed); + self.latency.on_send().await; } + Ok(()) } - /// Receive a message from the peer. - pub async fn receive_message(&mut self) -> NetworkResult> { - // If the state was cleared e.g. by a write-path broken pipe, treat as disconnected - // so the reader loop handles it identically to a read-path EOF. - let state_arc = self.state.as_ref().ok_or(NetworkError::PeerDisconnected)?; - - // Lock the state for the entire read operation - // This ensures no concurrent access to the socket - let mut state = state_arc.lock().await; - - // Buffered, stateful framing - const HEADER_LEN: usize = 24; // magic[4] + cmd[12] + length[4] + checksum[4] - const MAX_RESYNC_STEPS_PER_CALL: usize = 64; - - let result = async { - let magic_bytes = self.network.magic().to_le_bytes(); - let mut resync_steps = 0usize; - - loop { - // Ensure header availability - if state.framing_buffer.len() < HEADER_LEN { - match Self::read_some(&mut state).await { - Ok(0) => { - tracing::info!("Peer {} closed connection (EOF)", self.address); - return Err(NetworkError::PeerDisconnected); - } - Ok(_) => {} - Err(ref e) - if e.kind() == std::io::ErrorKind::ConnectionAborted - || e.kind() == std::io::ErrorKind::ConnectionReset => - { - tracing::info!("Peer {} connection reset/aborted", self.address); - return Err(NetworkError::PeerDisconnected); - } - Err(e) => { - return Err(NetworkError::ConnectionFailed(format!( - "Read failed: {}", - e - ))); - } - } - } - - // Align to magic - if state.framing_buffer.len() >= 4 && state.framing_buffer[..4] != magic_bytes { - if let Some(pos) = - state.framing_buffer.windows(4).position(|w| w == magic_bytes) - { - if pos > 0 { - tracing::warn!( - "{}: stream desync: skipping {} stray bytes before magic", - self.address, - pos - ); - state.framing_buffer.drain(0..pos); - resync_steps += 1; - if resync_steps >= MAX_RESYNC_STEPS_PER_CALL { - return Ok(None); - } - continue; - } - } else { - // Keep last 3 bytes of potential magic prefix - if state.framing_buffer.len() > 3 { - let dropped = state.framing_buffer.len() - 3; - tracing::warn!( - "{}: stream desync: dropping {} bytes (no magic found)", - self.address, - dropped - ); - state.framing_buffer.drain(0..dropped); - resync_steps += 1; - if resync_steps >= MAX_RESYNC_STEPS_PER_CALL { - return Ok(None); - } - } - // Need more data - match Self::read_some(&mut state).await { - Ok(0) => { - tracing::info!("Peer {} closed connection (EOF)", self.address); - return Err(NetworkError::PeerDisconnected); - } - Ok(_) => {} - Err(e) => { - return Err(NetworkError::ConnectionFailed(format!( - "Read failed: {}", - e - ))); - } - } - continue; - } - } - - // Ensure full header - if state.framing_buffer.len() < HEADER_LEN { - match Self::read_some(&mut state).await { - Ok(0) => { - tracing::info!("Peer {} closed connection (EOF)", self.address); - return Err(NetworkError::PeerDisconnected); - } - Ok(_) => {} - Err(e) => { - return Err(NetworkError::ConnectionFailed(format!( - "Read failed: {}", - e - ))); - } - } - continue; - } - - // Parse header fields - let length_le = u32::from_le_bytes([ - state.framing_buffer[16], - state.framing_buffer[17], - state.framing_buffer[18], - state.framing_buffer[19], - ]) as usize; - let header_checksum = [ - state.framing_buffer[20], - state.framing_buffer[21], - state.framing_buffer[22], - state.framing_buffer[23], - ]; - // Validate announced length to prevent unbounded accumulation or overflow - if length_le > dashcore::network::message::MAX_MSG_SIZE { - return Err(NetworkError::ProtocolError(format!( - "Declared payload length {} exceeds MAX_MSG_SIZE {}", - length_le, - dashcore::network::message::MAX_MSG_SIZE - ))); - } - let total_len = match HEADER_LEN.checked_add(length_le) { - Some(v) => v, - None => { - return Err(NetworkError::ProtocolError( - "Message length overflow".to_string(), - )); - } - }; - - // Ensure full frame available - if state.framing_buffer.len() < total_len { - match Self::read_some(&mut state).await { - Ok(0) => { - tracing::info!("Peer {} closed connection (EOF)", self.address); - return Err(NetworkError::PeerDisconnected); - } - Ok(_) => {} - Err(e) => { - return Err(NetworkError::ConnectionFailed(format!( - "Read failed: {}", - e - ))); - } - } - continue; - } - - // Verify checksum - let payload_slice = &state.framing_buffer[HEADER_LEN..total_len]; - let expected = { - let checksum = ::hash( - payload_slice, - ); - [checksum[0], checksum[1], checksum[2], checksum[3]] - }; - if expected != header_checksum { - tracing::warn!( - "Skipping message with invalid checksum from {}: expected {:02x?}, actual {:02x?}", - self.address, - expected, - header_checksum - ); - if header_checksum == [0, 0, 0, 0] { - tracing::warn!( - "All-zeros checksum detected from {}, likely corrupted stream - resyncing", - self.address - ); - } - // Resync by dropping a byte and retrying - state.framing_buffer.drain(0..1); - resync_steps += 1; - if resync_steps >= MAX_RESYNC_STEPS_PER_CALL { - return Ok(None); - } - continue; - } - - // Decode full RawNetworkMessage from the frame using existing decoder - let mut cursor = std::io::Cursor::new(&state.framing_buffer[..total_len]); - match RawNetworkMessage::consensus_decode(&mut cursor) { - Ok(raw_message) => { - // Consume bytes - state.framing_buffer.drain(0..total_len); - - // Validate magic matches our network - if raw_message.magic != self.network.magic() { - tracing::warn!( - "Received message with wrong magic bytes: expected {:#x}, got {:#x}", - self.network.magic(), - raw_message.magic - ); - return Err(NetworkError::ProtocolError(format!( - "Wrong magic bytes: expected {:#x}, got {:#x}", - self.network.magic(), - raw_message.magic - ))); - } - - tracing::trace!( - "Successfully decoded message from {}: {:?}", - self.address, - raw_message.payload.cmd() - ); - - return Ok(Some(Message::new(self.address, raw_message.payload))); - } - Err(e) => { - tracing::warn!( - "{}: decode error after framing ({}), attempting resync", - self.address, - e - ); - state.framing_buffer.drain(0..1); - resync_steps += 1; - if resync_steps >= MAX_RESYNC_STEPS_PER_CALL { - return Ok(None); - } - continue; - } - } - } + /// Note that `n` earlier pipeline requests have fully completed, freeing that + /// much in-flight work. Used for streaming responses (`getcfilters` -> many + /// `cfilter`s) that the reader can't attribute to a finished request on its + /// own; single-response requests are decremented directly in the reader. + pub(crate) async fn response_completed(&self, n: usize) { + let _ = self + .in_flight + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(n))); + for _ in 0..n { + self.latency.complete_one().await; } - .await; - - // Drop the lock before disconnecting - drop(state); + } - // Handle disconnection if needed - if let Err(NetworkError::PeerDisconnected) = &result { - self.state = None; - self.connected_at = None; + /// Reclaim the in-flight slots of requests this peer has not answered within + /// `timeout`, returning how many. Without this they leak: when a pipeline + /// times a request out it re-queues it, but nothing tells the peer its + /// request died, so the peer's `in_flight` counter stays raised forever. + /// Enough dead requests and EVERY peer sits at its cap with nothing actually + /// on the wire — the router sees zero free capacity, stops sending, and the + /// sync deadlocks with a full queue (reproduced at 32 peers: 0 MB/s, queue + /// 140, no sends). The count also identifies the peer as unresponsive, so the + /// caller can drop it rather than keep handing it work. + pub(crate) async fn reap_stale(&self, timeout: Duration) -> usize { + let n = self.latency.reap(timeout).await; + if n > 0 { + let _ = self + .in_flight + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(n))); } - - result + n } - /// Check if the connection is active. - pub fn is_connected(&self) -> bool { - self.state.is_some() + /// Per-peer response latency: (completed requests, average ms, worst ms). + pub(crate) fn latency_stats(&self) -> (u64, f64, f64) { + self.latency.snapshot() } - /// Check if connection appears healthy (not just connected). - pub fn is_healthy(&self) -> bool { - if !self.is_connected() { - tracing::debug!("Connection to {} marked unhealthy: not connected", self.address); - return false; - } - - let now = SystemTime::now(); - - // If we have exchanged pings/pongs, check the last activity - if let Some(last_pong) = self.last_pong_received { - if let Ok(duration) = now.duration_since(last_pong) { - // If no pong in 10 minutes, consider unhealthy - if duration > Duration::from_secs(600) { - tracing::warn!("Connection to {} marked unhealthy: no pong received for {} seconds (limit: 600)", - self.address, duration.as_secs()); - return false; - } - } - } else if let Some(connected_at) = self.connected_at { - // If we haven't received any pongs yet, check how long we've been connected - if let Ok(duration) = now.duration_since(connected_at) { - // Give new connections 5 minutes before considering them unhealthy - if duration > Duration::from_secs(300) { - tracing::warn!("Connection to {} marked unhealthy: no pong activity after {} seconds (limit: 300, last_ping_sent: {:?})", - self.address, duration.as_secs(), self.last_ping_sent.is_some()); - return false; - } - } - } - - // Connection is healthy - true + /// Cumulative (completed request count, total service-time ns) for this peer. + /// The bandwidth controller diffs these across a window to get the download + /// completion rate and service time that size the in-flight budget. + pub(crate) fn latency_totals(&self) -> (u64, u64) { + self.latency.totals() } - /// Get connection statistics. - pub fn stats(&self) -> (u64, u64) { - (self.bytes_sent, 0) // TODO: Track bytes received + /// Handshake round-trip latency in ms (0 if unmeasured). + pub(crate) fn lag_ms(&self) -> u32 { + self.lag_ms.load(Ordering::Relaxed) } - /// Send a ping message with a random nonce. - pub async fn send_ping(&mut self) -> NetworkResult { - let nonce = rand::random::(); - let ping_message = NetworkMessage::Ping(nonce); - - self.send_message(ping_message).await?; - - let now = SystemTime::now(); - self.last_ping_sent = Some(now); - self.pending_pings.insert(nonce, now); + /// Close this connection: cancel its reader so the socket shuts down. Used to + /// drop probed-but-unselected peers instead of leaking their readers. + pub(crate) fn close(&self) { + self.token.cancel(); + } - tracing::trace!("Sent ping to {} with nonce {}", self.address, nonce); + /// This peer's current measured in-flight capacity (its serving BDP). + pub(crate) fn cap(&self) -> usize { + self.cap.load(Ordering::Relaxed) + } - Ok(nonce) + /// Update this peer's measured in-flight capacity (called by the controller). + pub(crate) fn set_cap(&self, n: usize) { + self.cap.store(n, Ordering::Relaxed); } +} - /// Handle a received ping message by sending a pong response. - pub async fn handle_ping(&mut self, nonce: u64) -> NetworkResult<()> { - let pong_message = NetworkMessage::Pong(nonce); - self.send_message(pong_message).await?; +/// Pipeline requests we send and expect a response for (each adds one in-flight). +fn is_pipeline_request(msg: &NetworkMessage) -> bool { + match msg { + NetworkMessage::GetHeaders(_) + | NetworkMessage::GetHeaders2(_) + | NetworkMessage::GetCFHeaders(_) + | NetworkMessage::GetCFilters(_) => true, + // Block `getdata` is paced like any other request: the blocks pipeline + // sends ONE block per message, so a request is exactly one in-flight unit + // and one `block` in reply. Other `getdata` (chainlocks, islocks, txs) is + // control traffic — it carries several inventory items whose replies the + // reader cannot attribute one-for-one, so counting it would leak slots. + NetworkMessage::GetData(inv) => { + !inv.is_empty() + && inv + .iter() + .all(|i| matches!(i, dashcore::network::message_blockdata::Inventory::Block(_))) + } + _ => false, + } +} - tracing::debug!("Responded to ping from {} with pong nonce {}", self.address, nonce); +/// Single-message responses: the reader decrements one in-flight per message. +/// `cfilter` is excluded — one `getcfilters` yields up to 1000 `cfilter`s, so its +/// unit is freed once per batch by the filters pipeline via `response_completed`. +fn is_single_response(msg: &NetworkMessage) -> bool { + matches!( + msg, + NetworkMessage::Headers(_) + | NetworkMessage::Headers2(_) + | NetworkMessage::CFHeaders(_) + | NetworkMessage::Block(_) + ) +} - Ok(()) +impl DisconnectedPeer { + pub fn new(addr: SocketAddr, network: Network) -> Self { + DisconnectedPeer { + network, + addr, + } } - /// Handle a received pong message by validating the nonce. - pub fn handle_pong(&mut self, nonce: u64) -> NetworkResult<()> { - if let Some(sent_time) = self.pending_pings.remove(&nonce) { - let now = SystemTime::now(); - let rtt = now.duration_since(sent_time).unwrap_or(Duration::from_secs(0)); + pub fn addr(&self) -> SocketAddr { + self.addr + } - self.last_pong_received = Some(now); + #[allow(clippy::too_many_arguments)] + pub async fn connect( + self, + inbound: UnboundedSender, + shutdown: CancellationToken, + bytes: Arc, + ) -> NetworkResult { + let stream = TcpStream::connect(&self.addr).await.map_err(|e| { + NetworkError::ConnectionFailed(format!("Failed to connect to {}: {}", self.addr, e)) + })?; - tracing::debug!( - "Received valid pong from {} with nonce {} (RTT: {:?})", - self.address, - nonce, - rtt - ); + let (read_half, mut writer) = stream.into_split(); + let mut reader = FramedRead::new( + CountingReader { + inner: read_half, + bytes, + }, + RawNetworkMessageCodec, + ); + let magic = self.network.magic(); - Ok(()) - } else { - tracing::warn!("Received unexpected pong from {} with nonce {}", self.address, nonce); - Err(NetworkError::ProtocolError(format!( - "Unexpected pong nonce {} from {}", - nonce, self.address - ))) - } - } + handshake_send(&mut writer, magic, NetworkMessage::Version(build_version(self.addr))) + .await?; - /// Check if we need to send a ping (no ping/pong activity for 2 minutes). - pub fn should_ping(&self) -> bool { - let now = SystemTime::now(); + let deadline = tokio::time::Instant::now() + HANDSHAKE_TIMEOUT; + let mut peer_version: Option = None; + let mut got_verack = false; - // Check if we've sent a ping recently - if let Some(last_ping) = self.last_ping_sent { - if now.duration_since(last_ping).unwrap_or(Duration::MAX) < PING_INTERVAL { - return false; + while !(peer_version.is_some() && got_verack) { + let raw = match tokio::time::timeout_at(deadline, reader.next()).await { + Err(_) => return Err(NetworkError::Timeout), + Ok(None) => return Err(NetworkError::PeerDisconnected), + Ok(Some(Err(e))) => return Err(e.into()), + Ok(Some(Ok(raw))) => raw, + }; + if raw.magic != magic { + return Err(NetworkError::ProtocolError("wrong network magic".into())); } - } - - // Check if we've received a pong recently - if let Some(last_pong) = self.last_pong_received { - if now.duration_since(last_pong).unwrap_or(Duration::MAX) < PING_INTERVAL { - return false; + match raw.payload { + NetworkMessage::Version(v) => { + // BIP155: sendaddrv2 must be sent BEFORE verack. + handshake_send(&mut writer, magic, NetworkMessage::SendAddrV2).await?; + handshake_send(&mut writer, magic, NetworkMessage::Verack).await?; + peer_version = Some(v); + } + NetworkMessage::Verack => got_verack = true, + NetworkMessage::Ping(n) => { + handshake_send(&mut writer, magic, NetworkMessage::Pong(n)).await? + } + _ => {} } } - // If we haven't sent a ping or received a pong in 2 minutes, we should ping - true - } - - /// Remove pending pings that have timed out. - /// Returns `true` if any pings were removed. - pub fn remove_expired_pings(&mut self) -> bool { - const PING_TIMEOUT: Duration = Duration::from_secs(60); // 1 minute timeout for pings + let version = peer_version.ok_or(NetworkError::PeerDisconnected)?; - let now = SystemTime::now(); - let mut expired_nonces = Vec::new(); + // Announce sendheaders only after the handshake is fully complete. + handshake_send(&mut writer, magic, NetworkMessage::SendHeaders).await?; - for (&nonce, &sent_time) in &self.pending_pings { - if now.duration_since(sent_time).unwrap_or(Duration::ZERO) > PING_TIMEOUT { - expired_nonces.push(nonce); + // Measure round-trip lag with a post-handshake ping/pong. Sending a ping + // before the handshake completes makes some peers drop us, so we do it here. + let mut lag_ms: u32 = 0; + let ping_nonce: u64 = rand::random(); + let ping_sent = tokio::time::Instant::now(); + if handshake_send(&mut writer, magic, NetworkMessage::Ping(ping_nonce)).await.is_ok() { + let deadline = ping_sent + HANDSHAKE_TIMEOUT; + while let Ok(Some(Ok(raw))) = tokio::time::timeout_at(deadline, reader.next()).await { + if raw.magic != magic { + continue; + } + match raw.payload { + NetworkMessage::Pong(n) if n == ping_nonce => { + lag_ms = ping_sent.elapsed().as_millis().clamp(1, u32::MAX as u128) as u32; + break; + } + NetworkMessage::Ping(n) => { + let _ = handshake_send(&mut writer, magic, NetworkMessage::Pong(n)).await; + } + _ => {} + } } } - let has_expired = !expired_nonces.is_empty(); - for nonce in expired_nonces { - self.pending_pings.remove(&nonce); - tracing::warn!("Ping timeout for {} with nonce {}", self.address, nonce); - } - - has_expired - } - - /// Get ping/pong statistics. - pub fn ping_stats(&self) -> (Option, Option, usize) { - (self.last_ping_sent, self.last_pong_received, self.pending_pings.len()) - } + let writer = Arc::new(Mutex::new(writer)); + let in_flight = Arc::new(AtomicUsize::new(0)); + let latency = Arc::new(Latency::default()); + // Start with a tiny in-flight capacity; the controller grows it from this + // peer's measured serving rate. + let cap = Arc::new(AtomicUsize::new(2)); + // Per-connection token: cancelled by the global shutdown (parent) OR by + // `close()` to drop just this peer. + let token = shutdown.child_token(); + spawn_reader( + self.addr, + magic, + reader, + writer.clone(), + inbound, + in_flight.clone(), + latency.clone(), + token.clone(), + ); - /// Set that peer prefers headers2. - pub fn set_prefers_headers2(&mut self, prefers: bool) { - self.prefers_headers2 = prefers; - if prefers { - tracing::info!("Peer {} prefers headers2 compression", self.address); - } - } + tracing::debug!( + target: "dash_spv::network", + "peer connected: {} | lag={}ms height={}", + self.addr, + lag_ms, + version.start_height, + ); - /// Check if peer prefers headers2. - pub fn prefers_headers2(&self) -> bool { - self.prefers_headers2 + Ok(ConnectedPeer { + network: self.network, + addr: self.addr, + version, + lag_ms: AtomicU32::new(lag_ms), + in_flight, + latency, + writer, + cap, + token, + }) } +} - /// Set that peer sent us SendHeaders2. - pub fn set_peer_sent_sendheaders2(&mut self, sent: bool) { - self.sent_sendheaders2 = sent; - if sent { - tracing::info!( - "Peer {} sent SendHeaders2 - they will send compressed headers", - self.address - ); +#[allow(clippy::too_many_arguments)] +fn spawn_reader( + addr: SocketAddr, + magic: u32, + mut reader: PeerReader, + writer: Arc>, + inbound: UnboundedSender, + in_flight: Arc, + latency: Arc, + shutdown: CancellationToken, +) { + tokio::spawn(async move { + loop { + let t_recv = std::time::Instant::now(); + let next = tokio::select! { + _ = shutdown.cancelled() => break, + next = reader.next() => next, + }; + crate::timer::P_READ_RECV.add(t_recv.elapsed()); + match next { + None => break, + Some(Err(e)) => { + tracing::error!("NETWORK: reader {} stopped: {}", addr, e); + break; + } + Some(Ok(raw)) => { + if raw.magic != magic { + continue; + } + // A single-message response completes one unit of in-flight + // work. Streaming responses (cfilter) are freed per batch by + // the filters pipeline instead. + if is_single_response(&raw.payload) { + let _ = in_flight.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| { + Some(v.saturating_sub(1)) + }); + latency.complete_one().await; + } + match raw.payload { + NetworkMessage::Ping(nonce) => { + let pong = RawNetworkMessage { + magic, + payload: NetworkMessage::Pong(nonce), + }; + if writer + .lock() + .await + .write_all(&encode::serialize(&pong)) + .await + .is_err() + { + break; + } + } + payload => { + if inbound.send(PeerEvent::Message(addr, payload)).is_err() { + break; + } + } + } + } + } } - } - - /// Check if peer sent us SendHeaders2. - pub fn peer_sent_sendheaders2(&self) -> bool { - self.sent_sendheaders2 - } - /// Check if we can request headers2 from this peer. - pub fn can_request_headers2(&self) -> bool { - // We can request headers2 if peer has the service flag for headers2 support - // Note: We don't wait for SendHeaders2 from peer as that creates a race condition - // during initial sync. The service flag is sufficient to know they support headers2. - if let Some(services) = self.services { - dashcore::network::constants::ServiceFlags::from(services) - .has(dashcore::network::constants::NODE_HEADERS_COMPRESSED) - } else { - false - } - } + tracing::info!("NETWORK: peer {} disconnected", addr); + let _ = inbound.send(PeerEvent::Disconnected(addr)); + }); } -#[cfg(test)] -impl Peer { - pub(crate) fn set_services(&mut self, flags: ServiceFlags) { - self.services = Some(flags.as_u64()); - } +fn build_version(peer: SocketAddr) -> VersionMessage { + let now = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0); + let unspecified: SocketAddr = ([0u8, 0, 0, 0], 0).into(); + VersionMessage::new( + ServiceFlags::NONE, + now, + Address::new(&peer, ServiceFlags::NETWORK), + Address::new(&unspecified, ServiceFlags::NONE), + rand::random(), + USER_AGENT.to_string(), + 0, + false, + [0u8; 32], + ) } -#[cfg(test)] -mod tests { - use std::net::SocketAddr; - use std::time::{Duration, SystemTime}; - - use super::Peer; - - #[test] - fn remove_expired_pings() { - let addr: SocketAddr = "127.0.0.1:9999".parse().unwrap(); - let mut peer = Peer::dummy(addr); - let now = SystemTime::now(); - let expired = now - Duration::from_secs(61); - - // No pings at all - assert!(!peer.remove_expired_pings()); - - // Only recent pings — nothing removed - peer.pending_pings.insert(1, now); - peer.pending_pings.insert(2, now); - assert!(!peer.remove_expired_pings()); - assert_eq!(peer.pending_pings.len(), 2); - - // Add an expired ping — only it gets removed - peer.pending_pings.insert(3, expired); - assert!(peer.remove_expired_pings()); - assert_eq!(peer.pending_pings.len(), 2); - assert!(!peer.pending_pings.contains_key(&3)); - - // All expired — map ends up empty - peer.pending_pings.clear(); - peer.pending_pings.insert(10, expired); - peer.pending_pings.insert(20, expired); - assert!(peer.remove_expired_pings()); - assert!(peer.pending_pings.is_empty()); - } +async fn handshake_send( + writer: &mut OwnedWriteHalf, + magic: u32, + payload: NetworkMessage, +) -> NetworkResult<()> { + let raw = RawNetworkMessage { + magic, + payload, + }; + writer + .write_all(&encode::serialize(&raw)) + .await + .map_err(|e| NetworkError::ConnectionFailed(format!("handshake write failed: {}", e)))?; + writer + .flush() + .await + .map_err(|e| NetworkError::ConnectionFailed(format!("handshake flush failed: {}", e)))?; + Ok(()) } diff --git a/dash-spv/src/network/pool.rs b/dash-spv/src/network/pool.rs deleted file mode 100644 index 1a2e68b21..000000000 --- a/dash-spv/src/network/pool.rs +++ /dev/null @@ -1,316 +0,0 @@ -//! Peer pool for managing multiple peer connections - -use crate::error::{NetworkError, SpvError as Error}; -use crate::network::peer::Peer; -use dashcore::network::constants::ServiceFlags; -use dashcore::prelude::CoreBlockHeight; -use std::collections::{HashMap, HashSet}; -use std::net::SocketAddr; -use std::sync::Arc; -use tokio::sync::RwLock; - -/// Pool for managing multiple peer instances -pub struct PeerPool { - /// Active peers mapped by address - peers: Arc>>>>, - /// Addresses currently being connected to - connecting: Arc>>, - /// Maximum number of simultaneous peer connections (from `ClientConfig::max_peers`). - max_peers: usize, -} - -impl PeerPool { - /// Create a new peer pool with a connection cap. - pub fn new(max_peers: usize) -> Self { - // Assert peers are greater than 0. We may change this - // so 0 means 'connect to as many peers as you can' - debug_assert!(max_peers > 0, "max_peers must be greater than 0 for the spv client to sync"); - - Self { - peers: Arc::new(RwLock::new(HashMap::new())), - connecting: Arc::new(RwLock::new(HashSet::new())), - max_peers, - } - } - - /// Mark an address as being connected to - pub async fn mark_connecting(&self, addr: SocketAddr) -> bool { - let mut connecting = self.connecting.write().await; - connecting.insert(addr) - } - - /// Add a peer to the pool - pub async fn add_peer(&self, addr: SocketAddr, peer: Peer) -> Result<(), Error> { - let mut peers = self.peers.write().await; - let mut connecting = self.connecting.write().await; - - // Remove from connecting set - connecting.remove(&addr); - - // Check if we're at capacity - if peers.len() >= self.max_peers { - return Err(Error::Network(NetworkError::ConnectionFailed(format!( - "Maximum peers ({}) reached", - self.max_peers - )))); - } - - // Check if already connected - if peers.contains_key(&addr) { - return Err(Error::Network(NetworkError::ConnectionFailed(format!( - "Already connected to {}", - addr - )))); - } - - peers.insert(addr, Arc::new(RwLock::new(peer))); - tracing::info!("Added peer {}, total peers: {}", addr, peers.len()); - Ok(()) - } - - /// Remove a peer from the pool and clear connecting state - pub async fn remove_peer(&self, addr: &SocketAddr) -> Option>> { - self.connecting.write().await.remove(addr); - let removed = self.peers.write().await.remove(addr); - if removed.is_some() { - tracing::info!("Removed peer {}", addr); - } - removed - } - - /// Get all active peers - pub async fn get_all_peers(&self) -> Vec<(SocketAddr, Arc>)> { - self.peers.read().await.iter().map(|(addr, peer)| (*addr, peer.clone())).collect() - } - - /// Get a specific peer - pub async fn get_peer(&self, addr: &SocketAddr) -> Option>> { - self.peers.read().await.get(addr).cloned() - } - - /// Get the number of active peers - pub async fn peer_count(&self) -> usize { - self.peers.read().await.len() - } - - /// Check if connected to a specific peer - pub async fn is_connected(&self, addr: &SocketAddr) -> bool { - self.peers.read().await.contains_key(addr) - } - - /// Check if currently connecting to a peer - pub async fn is_connecting(&self, addr: &SocketAddr) -> bool { - self.connecting.read().await.contains(addr) - } - - /// Get all connected peer addresses - pub async fn get_connected_addresses(&self) -> Vec { - self.peers.read().await.keys().copied().collect() - } - - pub async fn get_best_height(&self) -> Option { - let peers = self.get_all_peers().await; - - if peers.is_empty() { - tracing::debug!("get_best_height: No peers available"); - return None; - } - - let mut best_height = 0u32; - let mut peer_count = 0; - - for (addr, peer) in peers.iter() { - let peer_guard = peer.read().await; - peer_count += 1; - - tracing::debug!( - "get_best_height: Peer {} - best_height: {:?}, version: {:?}, connected: {}", - addr, - peer_guard.best_height(), - peer_guard.version(), - peer_guard.is_connected(), - ); - - if let Some(peer_height) = peer_guard.best_height() { - if peer_height > 0 { - best_height = best_height.max(peer_height); - tracing::debug!( - "get_best_height: Updated best_height to {} from peer {}", - best_height, - addr - ); - } - } - } - - tracing::debug!( - "get_best_height: Checked {} peers, best_height: {}", - peer_count, - best_height - ); - - if best_height > 0 { - Some(best_height) - } else { - None - } - } - - /// Find the first connected peer that advertises the given service flags. - pub(crate) async fn peer_with_service( - &self, - flags: ServiceFlags, - ) -> Option<(SocketAddr, Arc>)> { - let peers = self.peers.read().await; - for (addr, peer) in peers.iter() { - if peer.read().await.has_service(flags) { - return Some((*addr, Arc::clone(peer))); - } - } - None - } - - /// Collect all connected peers that advertise the given service flags. - pub(crate) async fn peers_with_service( - &self, - flags: ServiceFlags, - ) -> Vec<(SocketAddr, Arc>)> { - let peers = self.peers.read().await; - let mut result = Vec::new(); - for (addr, peer) in peers.iter() { - if peer.read().await.has_service(flags) { - result.push((*addr, peer.clone())); - } - } - result - } - - /// Check whether any connected peer advertises the given service flags. - pub(crate) async fn has_peers_with_service(&self, flags: ServiceFlags) -> bool { - let peers = self.peers.read().await; - for peer in peers.values() { - if peer.read().await.has_service(flags) { - return true; - } - } - false - } - - /// Check if we need more peers - pub async fn needs_more_peers(&self) -> bool { - self.peer_count().await < self.max_peers - } - - /// Check if we can accept more peers - pub async fn can_accept_peers(&self) -> bool { - self.peer_count().await < self.max_peers - } - - /// Remove unhealthy peers and return their addresses so the caller can - /// emit the appropriate network events. - pub async fn remove_unhealthy(&self) -> Vec { - let peers = self.peers.read().await; - let mut unhealthy = Vec::new(); - - // Check each peer's health - for (addr, peer) in peers.iter() { - // Use blocking read to properly check health - let peer_guard = peer.read().await; - if !peer_guard.is_healthy() { - unhealthy.push(*addr); - } - } - - // Release read lock before taking write lock - drop(peers); - - // Remove unhealthy connections - if !unhealthy.is_empty() { - let mut peers = self.peers.write().await; - unhealthy.retain(|addr| peers.remove(addr).is_some()); - } - - unhealthy - } -} - -impl Default for PeerPool { - fn default() -> Self { - Self::new(8) - } -} - -#[cfg(test)] -impl PeerPool { - pub(crate) async fn insert_peer_with_services(&self, addr: SocketAddr, flags: ServiceFlags) { - let mut peer = Peer::dummy(addr); - peer.set_services(flags); - self.peers.write().await.insert(addr, Arc::new(RwLock::new(peer))); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_peer_pool_basic() { - let pool = PeerPool::new(8); - - // Initial state - assert_eq!(pool.peer_count().await, 0); - assert!(pool.needs_more_peers().await); - assert!(pool.can_accept_peers().await); - - // Test marking as connecting - let addr = "127.0.0.1:9999".parse().expect("Failed to parse test address"); - assert!(pool.mark_connecting(addr).await); - assert!(!pool.mark_connecting(addr).await); // Already marked - assert!(pool.is_connecting(&addr).await); - } - - #[tokio::test] - async fn test_service_lookup() { - let pool = PeerPool::new(8); - let compact_filters = ServiceFlags::COMPACT_FILTERS; - let combined = compact_filters | ServiceFlags::NODE_HEADERS_COMPRESSED; - - // No matches on empty pool - assert!(pool.peer_with_service(compact_filters).await.is_none()); - assert!(pool.peers_with_service(compact_filters).await.is_empty()); - - // No matches when peers lack the requested flag - let addr1: SocketAddr = "127.0.0.1:1001".parse().unwrap(); - pool.insert_peer_with_services(addr1, ServiceFlags::NETWORK).await; - assert!(pool.peer_with_service(compact_filters).await.is_none()); - assert!(pool.peers_with_service(compact_filters).await.is_empty()); - - // Single-flag lookup returns matching peers - let addr2: SocketAddr = "127.0.0.1:1002".parse().unwrap(); - let addr3: SocketAddr = "127.0.0.1:1003".parse().unwrap(); - pool.insert_peer_with_services(addr2, ServiceFlags::NETWORK | compact_filters).await; - pool.insert_peer_with_services(addr3, ServiceFlags::NETWORK | combined).await; - - let (found_addr, found_peer) = pool.peer_with_service(compact_filters).await.unwrap(); - assert!(found_addr == addr2 || found_addr == addr3); - assert!(found_peer.read().await.has_service(compact_filters)); - - let filter_peers: HashMap = - pool.peers_with_service(compact_filters).await.into_iter().collect(); - assert_eq!(filter_peers.len(), 2); - assert!(filter_peers.contains_key(&addr2)); - assert!(filter_peers.contains_key(&addr3)); - - // Combined flags require all bits present - let (found_addr, _) = pool.peer_with_service(combined).await.unwrap(); - assert_eq!(found_addr, addr3); - let combined_peers = pool.peers_with_service(combined).await; - assert_eq!(combined_peers.len(), 1); - assert_eq!(combined_peers[0].0, addr3); - - // NONE matches every peer in the pool - assert!(pool.peer_with_service(ServiceFlags::NONE).await.is_some()); - let all = pool.peers_with_service(ServiceFlags::NONE).await; - assert_eq!(all.len(), 3); - } -} diff --git a/dash-spv/src/network/reputation.rs b/dash-spv/src/network/reputation.rs deleted file mode 100644 index f90656584..000000000 --- a/dash-spv/src/network/reputation.rs +++ /dev/null @@ -1,439 +0,0 @@ -//! Peer reputation management system -//! -//! This module implements a reputation system to track peer behavior and protect -//! against malicious peers. It tracks both positive and negative behaviors, -//! implements automatic banning for excessive misbehavior, and provides reputation -//! decay over time for recovery. - -use crate::storage::PeerStorage; -use dashcore::network::address::AddrV2Message; -use serde::{Deserialize, Deserializer, Serialize}; -use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::RwLock; - -/// Reason for a peer reputation change. Each reason owns its score delta -/// (positive = penalty, negative = reward) and a human-readable label. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ChangeReason { - HandshakeFailed, - ConnectionFailed, - Headers2DecompressionFailed, - ReadTimeout, - PingFailed, - InvalidTransactionInBlock, - ManuallyBanned, - LongUptime, -} - -impl ChangeReason { - /// Score delta for this reason: positive for misbehavior (penalty), - /// negative for good behavior (reward). - pub fn score(&self) -> i32 { - match self { - ChangeReason::HandshakeFailed => 10, - ChangeReason::ConnectionFailed => 2, - ChangeReason::Headers2DecompressionFailed => 10, - ChangeReason::ReadTimeout => 5, - ChangeReason::PingFailed => 5, - ChangeReason::InvalidTransactionInBlock => 20, - ChangeReason::ManuallyBanned => 100, - ChangeReason::LongUptime => -5, - } - } -} - -impl std::fmt::Display for ChangeReason { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let label = match self { - ChangeReason::HandshakeFailed => "Handshake failed", - ChangeReason::ConnectionFailed => "Connection failed", - ChangeReason::Headers2DecompressionFailed => "Headers2 decompression failed", - ChangeReason::ReadTimeout => "Read timeout", - ChangeReason::PingFailed => "Ping failed", - ChangeReason::InvalidTransactionInBlock => "Invalid transaction type in block", - ChangeReason::ManuallyBanned => "Manually banned", - ChangeReason::LongUptime => "Long connection uptime", - }; - f.write_str(label) - } -} - -/// Ban duration for misbehaving peers -const BAN_DURATION: Duration = Duration::from_secs(24 * 60 * 60); // 24 hours - -/// Reputation decay interval -const DECAY_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour - -/// Amount to decay reputation score per interval -const DECAY_AMOUNT: i32 = 5; - -/// Maximum misbehavior score before a peer is banned -const MAX_MISBEHAVIOR_SCORE: i32 = 100; - -/// Minimum score (most positive reputation) -const MIN_MISBEHAVIOR_SCORE: i32 = -50; - -const MAX_BAN_COUNT: u32 = 1000; - -const MAX_ACTION_COUNT: u64 = 1_000_000; - -fn clamp_peer_score<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let mut v = i32::deserialize(deserializer)?; - - if v < MIN_MISBEHAVIOR_SCORE { - tracing::warn!("Peer has invalid score {v}, clamping to min {MIN_MISBEHAVIOR_SCORE}"); - v = MIN_MISBEHAVIOR_SCORE - } else if v > MAX_MISBEHAVIOR_SCORE { - tracing::warn!("Peer has invalid score {v}, clamping to max {MAX_MISBEHAVIOR_SCORE}"); - v = MAX_MISBEHAVIOR_SCORE - } - - Ok(v) -} - -fn clamp_peer_ban_count<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let mut v = u32::deserialize(deserializer)?; - - if v > MAX_BAN_COUNT { - tracing::warn!("Peer has excessive ban count {v}, clamping to {MAX_BAN_COUNT}"); - v = MAX_BAN_COUNT - } - - Ok(v) -} - -fn clamp_peer_connection_attempts<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let mut v = u64::deserialize(deserializer)?; - - v = v.min(MAX_ACTION_COUNT); - - Ok(v) -} - -/// Peer reputation entry -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PeerReputation { - /// Current misbehavior score - #[serde(deserialize_with = "clamp_peer_score")] - pub score: i32, - - /// Number of times this peer has been banned - #[serde(deserialize_with = "clamp_peer_ban_count")] - pub ban_count: u32, - - /// Time when the peer was banned (if currently banned) - #[serde(skip)] - pub banned_until: Option, - - /// Last time the reputation was updated - #[serde(skip, default = "Instant::now")] - pub last_update: Instant, - - /// Total number of positive actions - pub positive_actions: u64, - - /// Total number of negative actions - pub negative_actions: u64, - - /// Connection count - #[serde(deserialize_with = "clamp_peer_connection_attempts")] - pub connection_attempts: u64, - - /// Successful connection count - pub successful_connections: u64, - - /// Last connection time - #[serde(skip)] - pub last_connection: Option, -} - -impl Default for PeerReputation { - fn default() -> Self { - Self { - score: 0, - ban_count: 0, - banned_until: None, - last_update: Instant::now(), - positive_actions: 0, - negative_actions: 0, - connection_attempts: 0, - successful_connections: 0, - last_connection: None, - } - } -} - -impl PeerReputation { - /// Check if the peer is currently banned - pub fn is_banned(&self) -> bool { - self.banned_until.is_some_and(|until| Instant::now() < until) - } - - /// Get remaining ban time - pub fn ban_time_remaining(&self) -> Option { - self.banned_until.and_then(|until| { - let now = Instant::now(); - if now < until { - Some(until - now) - } else { - None - } - }) - } - - /// Apply reputation decay - pub fn apply_decay(&mut self) { - let now = Instant::now(); - let elapsed = now - self.last_update; - - // Apply decay for each interval that has passed - let intervals = elapsed.as_secs() / DECAY_INTERVAL.as_secs(); - if intervals > 0 { - // Use saturating conversion to prevent overflow - // Cap at a reasonable maximum to avoid excessive decay - let intervals_i32 = intervals.min(i32::MAX as u64) as i32; - let decay = intervals_i32.saturating_mul(DECAY_AMOUNT); - self.score = (self.score - decay).max(MIN_MISBEHAVIOR_SCORE); - self.last_update = now; - } - - // Check if ban has expired - if self.is_banned() && self.ban_time_remaining().is_none() { - self.banned_until = None; - } - } -} - -/// Peer reputation manager -pub struct PeerReputationManager { - /// Reputation data for each peer - reputations: Arc>>, -} - -impl Default for PeerReputationManager { - fn default() -> Self { - Self::new() - } -} - -impl PeerReputationManager { - /// Create a new reputation manager - pub fn new() -> Self { - Self { - reputations: Arc::new(RwLock::new(HashMap::new())), - } - } - - /// Update peer reputation by the score delta of `reason`. - pub async fn update_reputation(&self, peer: SocketAddr, reason: ChangeReason) -> bool { - let score_change = reason.score(); - - let mut reputations = self.reputations.write().await; - let reputation = reputations.entry(peer).or_default(); - - // Apply decay first - reputation.apply_decay(); - - // Update score - let old_score = reputation.score; - reputation.score = - (reputation.score + score_change).clamp(MIN_MISBEHAVIOR_SCORE, MAX_MISBEHAVIOR_SCORE); - - // Track positive/negative actions - if score_change > 0 { - reputation.negative_actions += 1; - } else if score_change < 0 { - reputation.positive_actions += 1; - } - - // Check if peer should be banned - let should_ban = reputation.score >= MAX_MISBEHAVIOR_SCORE && !reputation.is_banned(); - if should_ban { - reputation.banned_until = Some(Instant::now() + BAN_DURATION); - reputation.ban_count += 1; - tracing::warn!( - "Peer {} banned for misbehavior (score: {}, ban #{}, reason: {})", - peer, - reputation.score, - reputation.ban_count, - reason - ); - } - - // Log significant changes - if score_change.abs() >= 10 || should_ban { - tracing::info!( - "Peer {} reputation changed: {} -> {} (change: {}, reason: {})", - peer, - old_score, - reputation.score, - score_change, - reason - ); - } - - should_ban - } - - /// Check if a peer is banned - pub async fn is_banned(&self, peer: &SocketAddr) -> bool { - let mut reputations = self.reputations.write().await; - if let Some(reputation) = reputations.get_mut(peer) { - reputation.apply_decay(); - reputation.is_banned() - } else { - false - } - } - - /// Record a connection attempt - pub async fn record_connection_attempt(&self, peer: SocketAddr) { - let mut reputations = self.reputations.write().await; - let reputation = reputations.entry(peer).or_default(); - reputation.connection_attempts += 1; - reputation.last_connection = Some(Instant::now()); - } - - /// Record a successful connection - pub async fn record_successful_connection(&self, peer: SocketAddr) { - let mut reputations = self.reputations.write().await; - let reputation = reputations.entry(peer).or_default(); - reputation.successful_connections += 1; - } - - /// Get all peer reputations - pub async fn get_all_reputations(&self) -> HashMap { - let mut reputations = self.reputations.write().await; - - // Apply decay to all peers - for reputation in reputations.values_mut() { - reputation.apply_decay(); - } - - reputations.clone() - } - - /// Clear banned status for a peer (admin function) - pub async fn unban_peer(&self, peer: &SocketAddr) { - let mut reputations = self.reputations.write().await; - if let Some(reputation) = reputations.get_mut(peer) { - reputation.banned_until = None; - reputation.score = reputation.score.min(MAX_MISBEHAVIOR_SCORE - 10); - tracing::info!("Manually unbanned peer {}", peer); - } - } - - /// Save reputation data to persistent storage - pub async fn save_to_storage(&self, storage: &impl PeerStorage) -> std::io::Result<()> { - let reputations = self.reputations.read().await; - - storage.save_peers_reputation(&reputations).await.map_err(std::io::Error::other) - } - - /// Load reputation data from persistent storage - pub async fn load_from_storage(&self, storage: &impl PeerStorage) -> std::io::Result<()> { - let data = storage.load_peers_reputation().await.map_err(std::io::Error::other)?; - - let mut reputations = self.reputations.write().await; - let mut loaded_count = 0; - let mut skipped_count = 0; - - for (addr, mut reputation) in data { - // Validate successful connections don't exceed attempts - reputation.successful_connections = - reputation.successful_connections.min(reputation.connection_attempts); - - // Skip entry if data appears corrupted - if reputation.positive_actions > MAX_ACTION_COUNT - || reputation.negative_actions > MAX_ACTION_COUNT - { - tracing::warn!("Skipping peer {} with potentially corrupted action counts", addr); - skipped_count += 1; - continue; - } - - // Apply initial decay based on ban count - if reputation.ban_count > 0 { - reputation.score = reputation.score.max(50); // Start with higher score for previously banned peers - } - - reputations.insert(addr, reputation); - loaded_count += 1; - } - - tracing::info!( - "Loaded reputation data for {} peers (skipped {} corrupted entries)", - loaded_count, - skipped_count - ); - Ok(()) - } -} - -/// Helper trait for reputation-aware peer selection -pub trait ReputationAware { - /// Select best peers based on reputation - fn select_best_peers( - &self, - available_peers: Vec, - count: usize, - ) -> impl std::future::Future> + Send; - - /// Check if we should connect to a peer based on reputation - fn should_connect_to_peer( - &self, - peer: &SocketAddr, - ) -> impl std::future::Future + Send; -} - -impl ReputationAware for PeerReputationManager { - async fn select_best_peers( - &self, - available_peers: Vec, - count: usize, - ) -> Vec { - let mut peer_scores = Vec::new(); - let mut reputations = self.reputations.write().await; - - for peer in available_peers { - let Ok(socket_addr) = peer.socket_addr() else { - tracing::warn!("Skip invalid peer address: {:?}", peer); - continue; - }; - - let reputation = reputations.entry(socket_addr).or_default(); - reputation.apply_decay(); - - if !reputation.is_banned() { - peer_scores.push((socket_addr, reputation.score)); - } - } - - // Sort by score (lower is better) - peer_scores.sort_by_key(|(_, score)| *score); - - // Return the best peers - peer_scores.into_iter().take(count).map(|(peer, _)| peer).collect() - } - - async fn should_connect_to_peer(&self, peer: &SocketAddr) -> bool { - !self.is_banned(peer).await - } -} - -// Include tests module -#[cfg(test)] -#[path = "reputation_tests.rs"] -mod reputation_tests; diff --git a/dash-spv/src/network/reputation_tests.rs b/dash-spv/src/network/reputation_tests.rs deleted file mode 100644 index 68b74e13b..000000000 --- a/dash-spv/src/network/reputation_tests.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Unit tests for reputation system (in-module tests) - -#[cfg(test)] -mod tests { - use crate::storage::{PersistentPeerStorage, PersistentStorage}; - - use super::super::*; - use std::net::SocketAddr; - - async fn score(manager: &PeerReputationManager, peer: &SocketAddr) -> i32 { - manager.get_all_reputations().await.get(peer).map_or(0, |rep| rep.score) - } - - #[tokio::test] - async fn test_basic_reputation_operations() { - let manager = PeerReputationManager::new(); - let peer: SocketAddr = "127.0.0.1:8333".parse().unwrap(); - - assert_eq!(score(&manager, &peer).await, 0); - - manager.update_reputation(peer, ChangeReason::HandshakeFailed).await; - assert_eq!(score(&manager, &peer).await, 10); - - manager.update_reputation(peer, ChangeReason::LongUptime).await; - assert_eq!(score(&manager, &peer).await, 5); - } - - #[tokio::test] - async fn test_banning_mechanism() { - let manager = PeerReputationManager::new(); - let peer: SocketAddr = "192.168.1.1:8333".parse().unwrap(); - - // Banned on the 10th violation (10 * 10 = 100). - for i in 0..10 { - let banned = manager.update_reputation(peer, ChangeReason::HandshakeFailed).await; - if i == 9 { - assert!(banned); - } else { - assert!(!banned); - } - } - - assert!(manager.is_banned(&peer).await); - } - - #[tokio::test] - async fn test_reputation_persistence() { - let manager = PeerReputationManager::new(); - let peer1: SocketAddr = "10.0.0.1:8333".parse().unwrap(); - let peer2: SocketAddr = "10.0.0.2:8333".parse().unwrap(); - - manager.update_reputation(peer1, ChangeReason::LongUptime).await; - manager.update_reputation(peer1, ChangeReason::LongUptime).await; - manager.update_reputation(peer2, ChangeReason::InvalidTransactionInBlock).await; - - let temp_dir = tempfile::TempDir::new().unwrap(); - let peer_storage = PersistentPeerStorage::open(temp_dir.path()) - .await - .expect("Failed to open PersistentPeerStorage"); - manager.save_to_storage(&peer_storage).await.unwrap(); - - let new_manager = PeerReputationManager::new(); - new_manager.load_from_storage(&peer_storage).await.unwrap(); - - assert_eq!(score(&new_manager, &peer1).await, -10); - assert_eq!(score(&new_manager, &peer2).await, 20); - } - - #[tokio::test] - async fn test_peer_selection() { - let manager = PeerReputationManager::new(); - - let good_peer = AddrV2Message::dummy(0, "1.1.1.1".parse().unwrap(), 8333); - let neutral_peer = AddrV2Message::dummy(0, "2.2.2.2".parse().unwrap(), 8333); - let bad_peer = AddrV2Message::dummy(0, "3.3.3.3".parse().unwrap(), 8333); - - manager.update_reputation(good_peer.socket_addr().unwrap(), ChangeReason::LongUptime).await; - manager - .update_reputation( - bad_peer.socket_addr().unwrap(), - ChangeReason::InvalidTransactionInBlock, - ) - .await; - - let all_peers = vec![good_peer.clone(), neutral_peer.clone(), bad_peer.clone()]; - let selected = manager.select_best_peers(all_peers, 2).await; - - assert_eq!(selected.len(), 2); - assert_eq!(selected[0], good_peer.socket_addr().unwrap()); - assert_eq!(selected[1], neutral_peer.socket_addr().unwrap()); - } - - #[tokio::test] - async fn test_connection_tracking() { - let manager = PeerReputationManager::new(); - let peer: SocketAddr = "127.0.0.1:9999".parse().unwrap(); - - // Track connection attempts - manager.record_connection_attempt(peer).await; - manager.record_connection_attempt(peer).await; - manager.record_successful_connection(peer).await; - - let reputations = manager.get_all_reputations().await; - let rep = &reputations[&peer]; - - assert_eq!(rep.connection_attempts, 2); - assert_eq!(rep.successful_connections, 1); - } -} diff --git a/dash-spv/src/network/tests.rs b/dash-spv/src/network/tests.rs deleted file mode 100644 index 4bb3447e0..000000000 --- a/dash-spv/src/network/tests.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! Unit tests for network module - -#[cfg(test)] -mod peer_tests { - use crate::network::peer::Peer; - use dashcore::Network; - use std::time::Duration; - - #[test] - fn test_peer_creation() { - let addr = "127.0.0.1:9999".parse().unwrap(); - let timeout = Duration::from_secs(30); - let peer = Peer::new(addr, timeout, Network::Mainnet); - - assert!(!peer.is_connected()); - assert_eq!(peer.address(), addr); - } -} - -#[cfg(test)] -mod pool_tests { - use crate::network::manager::PeerNetworkManager; - use crate::network::peer::Peer; - use crate::network::pool::PeerPool; - use crate::test_utils::test_socket_address; - use dashcore::network::constants::ServiceFlags; - use dashcore::Network; - use tokio::time::Duration; - - #[tokio::test] - async fn test_pool_limits() { - let pool = PeerPool::new(8); - - // Test needs_more_peers logic - assert!(pool.needs_more_peers().await); - - // Can accept up to 8 peers - assert!(pool.can_accept_peers().await); - - // Test peer count - assert_eq!(pool.peer_count().await, 0); - } - - #[tokio::test] - async fn test_capability_policy_for_handshake_and_eviction() { - let cf = ServiceFlags::COMPACT_FILTERS; - let mut incapable = - Peer::new(test_socket_address(9), Duration::from_secs(10), Network::Testnet); - incapable.set_services(ServiceFlags::NETWORK); - - // Handshake admission: keep fallback when no capable peer exists yet. - let manager = PeerNetworkManager::new_for_test(cf).await; - assert!(!manager.test_has_capable_peer().await); - assert!(!manager.test_should_reject_after_handshake(&incapable).await); - - // Handshake admission: reject incapable peers once a capable peer exists. - let manager = PeerNetworkManager::new_for_test(cf).await; - manager.insert_test_peer(test_socket_address(1), cf).await; - assert!(manager.test_has_capable_peer().await); - assert!(manager.test_should_reject_after_handshake(&incapable).await); - - // Healthy pool: all peers match, nothing evicted - let manager = PeerNetworkManager::new_for_test(cf).await; - manager.insert_test_peer(test_socket_address(1), cf).await; - manager.insert_test_peer(test_socket_address(2), cf).await; - manager.insert_test_peer(test_socket_address(3), cf).await; - manager.evict_mismatched_peers().await; - assert_eq!(manager.test_peer_count().await, 3); - - // Lone mismatched peer is preserved (never drop to zero) - let manager = PeerNetworkManager::new_for_test(cf).await; - manager.insert_test_peer(test_socket_address(1), ServiceFlags::NETWORK).await; - manager.evict_mismatched_peers().await; - assert_eq!(manager.test_peer_count().await, 1); - - // All peers lack service: tick 1 drops all but 1, tick 2 preserves the lone peer - let manager = PeerNetworkManager::new_for_test(cf).await; - manager.insert_test_peer(test_socket_address(1), ServiceFlags::NETWORK).await; - manager.insert_test_peer(test_socket_address(2), ServiceFlags::NETWORK).await; - manager.insert_test_peer(test_socket_address(3), ServiceFlags::NETWORK).await; - manager.evict_mismatched_peers().await; - assert_eq!(manager.test_peer_count().await, 1); - manager.evict_mismatched_peers().await; - assert_eq!(manager.test_peer_count().await, 1); - - // Mixed pool: only mismatched peers are dropped, matching peers survive - let manager = PeerNetworkManager::new_for_test(cf).await; - let p1 = test_socket_address(1); - let p2 = test_socket_address(2); - let p3 = test_socket_address(3); - let p4 = test_socket_address(4); - manager.insert_test_peer(p1, cf).await; - manager.insert_test_peer(p2, cf).await; - manager.insert_test_peer(p3, ServiceFlags::NETWORK).await; - manager.insert_test_peer(p4, ServiceFlags::NETWORK).await; - manager.evict_mismatched_peers().await; - assert_eq!(manager.test_peer_count().await, 2); - assert!(manager.test_is_connected(&p1).await); - assert!(manager.test_is_connected(&p2).await); - assert!(!manager.test_is_connected(&p3).await); - assert!(!manager.test_is_connected(&p4).await); - } - - #[tokio::test(start_paused = true)] - async fn test_capability_rejection_cache_expires() { - let manager = PeerNetworkManager::new_for_test(ServiceFlags::COMPACT_FILTERS).await; - let fresh = test_socket_address(42); - let expired = test_socket_address(43); - - manager.insert_test_capability_rejected(expired).await; - tokio::time::advance(Duration::from_secs(31 * 60)).await; - manager.insert_test_capability_rejected(fresh).await; - - assert!(manager.test_is_capability_rejected(&fresh).await); - assert!(!manager.test_is_capability_rejected(&expired).await); - - assert_eq!(manager.test_capability_rejected_count().await, 1); - } -} diff --git a/dash-spv/src/network2/discovery.rs b/dash-spv/src/network2/discovery.rs deleted file mode 100644 index 8d144153b..000000000 --- a/dash-spv/src/network2/discovery.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::net::SocketAddr; - -use dashcore::Network; -use rand::seq::SliceRandom; - -use crate::network::discovery::DnsDiscovery; -use crate::network2::peer::DisconnectedPeer; -use crate::ClientConfig; - -pub struct PeerDiscoverer { - network: Network, - // Empty means "use DNS discovery" - fixed: Vec, - dns_discovered: Option>, - dns: DnsDiscovery, -} - -impl PeerDiscoverer { - pub fn new(config: &ClientConfig) -> PeerDiscoverer { - PeerDiscoverer { - network: config.network, - fixed: config.peers.clone(), - dns_discovered: None, - dns: DnsDiscovery::new(), - } - } - - pub async fn get(&mut self, count: usize) -> Vec { - let addrs: Vec = if !self.fixed.is_empty() { - self.fixed.choose_multiple(&mut rand::thread_rng(), count).copied().collect() - } else { - let peers = match self.dns_discovered.as_mut() { - Some(known) => known, - None => { - let mut discovered = Self::dns_discover_peers(self.network).await; - discovered.shuffle(&mut rand::thread_rng()); - self.dns_discovered.insert(discovered) - } - }; - - let take = count.min(peers.len()); - peers.drain(..take).collect() - }; - - addrs.into_iter().map(|addr| DisconnectedPeer::new(addr, self.network)).collect() - } - - async fn dns_discover_peers(network: Network) -> Vec { - let seeds = network.dns_seeds(); - let port = network.default_p2p_port(); - let mut addresses = dash_network_seeds::addresses(network); - - for seed in seeds { - match tokio::net::lookup_host((*seed, port)).await { - Ok(iter) => { - let resolved: Vec = iter.collect(); - tracing::info!("DNS seed {} returned {} addresses", seed, resolved.len()); - addresses.extend(resolved); - } - Err(e) => { - tracing::warn!("Failed to resolve DNS seed {} (backup source): {}", seed, e); - } - } - } - - addresses.sort(); - addresses.dedup(); - - addresses - } -} diff --git a/dash-spv/src/network2/manager.rs b/dash-spv/src/network2/manager.rs deleted file mode 100644 index e378a54f8..000000000 --- a/dash-spv/src/network2/manager.rs +++ /dev/null @@ -1,1036 +0,0 @@ -use std::{ - collections::{HashMap, VecDeque}, - net::SocketAddr, - sync::{ - atomic::{AtomicU64, AtomicUsize, Ordering}, - Arc, - }, - time::Duration, -}; - -use dashcore::network::message::NetworkMessage; -use dashcore::network::message_blockdata::Inventory; -use futures::future::join_all; -use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; -use tokio::sync::{broadcast, Mutex, Notify}; -use tokio::task::JoinHandle; -use tokio_util::sync::CancellationToken; - -const HIGH_DEMAND_QUEUE: usize = 40; -const ADD_PEERS_BATCH: usize = 4; -const PROBE_BATCH: usize = 256; -/// Safety ceiling on how many peers backpressure may auto-connect. `max_peers` -/// (config) is the base/initial set; when the send queue stays backed up the -/// peers can't serve fast enough, so we recruit more — up to this cap — to raise -/// aggregate serving capacity. -/// -/// Kept near the empirical sweet spot: on testnet, throughput improves up to -/// ~16 peers, is flat-to-worse by 24-32, and OUTRIGHT STALLS around 64 — dozens -/// of slow peers each pin their measured in-flight on batches that never finish, -/// so there are no completions to wake the router. More peers past the knee only -/// adds congestion, not download rate (the peers, not our link, are the limit). -const MAX_CONNECTED_PEERS: usize = 16; - -/// How long the router waits on a full-capacity stall before checking whether the -/// "in-flight" requests holding that capacity are actually dead. -const STALL_CHECK: Duration = Duration::from_secs(5); - -/// A request unanswered for this long has its slot reclaimed: the owning pipeline -/// has long since timed it out and re-queued it elsewhere, so the slot holds -/// nothing. -/// -/// Generous, because slow is not dead. Blocks are paced through the same budget, -/// and a 2MB block over a weak link legitimately takes a while — the point is to -/// free capacity, not to punish peers (whether the peer is dropped is decided -/// separately, on whether it has EVER answered). -const STALE_REQUEST: Duration = Duration::from_secs(90); -use crate::{ - network2::{ - discovery::PeerDiscoverer, - peer::{ConnectedPeer, DisconnectedPeer, PeerEvent}, - }, - ClientConfig, -}; - -/// An inbound message on its way to the managers that subscribed to its type. -/// -/// Shared rather than cloned: a `block` or `cfilter` carries its whole payload, and the -/// pump would otherwise deep-copy it for every subscriber. See `spawn_pump`'s fan-out. -type Inbound = (SocketAddr, Arc); -type Subscribers = Arc>>>>; - -/// The kinds of peer message a sync manager can subscribe to. Replaces the -/// stringly-typed command names: managers declare interest with these variants -/// and the pump routes incoming messages by mapping `cmd()` back to one. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum MessageType { - Headers, - Inv, - CfHeaders, - CFilter, - Block, - MnListDiff, - QrInfo, - Tx, - IsDLock, - ChainLock, -} - -impl MessageType { - /// The wire command string this type corresponds to. - pub fn cmd(self) -> &'static str { - match self { - MessageType::Headers => "headers", - MessageType::Inv => "inv", - MessageType::CfHeaders => "cfheaders", - MessageType::CFilter => "cfilter", - MessageType::Block => "block", - MessageType::MnListDiff => "mnlistdiff", - MessageType::QrInfo => "qrinfo", - MessageType::Tx => "tx", - MessageType::IsDLock => "isdlock", - MessageType::ChainLock => "clsig", - } - } - - /// Map an incoming message's command back to a subscribed type, if any. - pub fn from_cmd(cmd: &str) -> Option { - Some(match cmd { - "headers" => MessageType::Headers, - "inv" => MessageType::Inv, - "cfheaders" => MessageType::CfHeaders, - "cfilter" => MessageType::CFilter, - "block" => MessageType::Block, - "mnlistdiff" => MessageType::MnListDiff, - "qrinfo" => MessageType::QrInfo, - "tx" => MessageType::Tx, - "isdlock" => MessageType::IsDLock, - "clsig" => MessageType::ChainLock, - _ => return None, - }) - } -} - -#[derive(Clone, Debug)] -pub enum NetworkEvent { - PeersUpdated, - PeerConnected(SocketAddr), - PeerDisconnected(SocketAddr), - /// A pipeline request just left the router for a peer (it is now on the - /// wire). Pipelines use this to start the request's response timeout from the - /// actual send, not from when it was queued (which fires spuriously when the - /// router is backed up). - RequestOnFlight(RequestKey), -} - -/// Identifies a pipeline request that has been sent, so the owning pipeline can -/// match it to its in-flight entry and start the timeout. One variant per -/// router-paced request type. -#[derive(Clone, Debug)] -pub enum RequestKey { - /// `getheaders` — keyed by the locator's first hash (the segment tip). - Headers(dashcore::BlockHash), - /// `getcfheaders` — keyed by the stop hash. - CfHeaders(dashcore::BlockHash), - /// `getcfilters` — keyed by the start height. - CFilters(u32), - /// `getmnlistdiff` — keyed by the target block hash. - MnListDiff(dashcore::BlockHash), - /// Block `getdata` — keyed by the requested block hash. - Block(dashcore::BlockHash), -} - -pub struct PeerNetworkManager { - connected_peers: Arc>>, - other_peers: Arc>>, - banned_peers: Vec, - discoverer: PeerDiscoverer, - msg_queue: Arc, - router: JoinHandle<()>, - pump: JoinHandle<()>, - inbound_tx: UnboundedSender, - subscribers: Subscribers, - events_tx: broadcast::Sender, - best_tip: u32, - max_peers: usize, - // Cancelled by `stop()` to tear down the router, pump and every peer reader. - shutdown: CancellationToken, -} - -/// What kind of work a queued message is, for scheduling. A single FIFO let one -/// pipeline monopolise the router: a filters phase queues batches by the hundred, -/// so a `getcfheaders` (or a small control message) landing behind them waited for -/// the whole backlog — the phases then advance one after another instead of -/// together. The router drains `Other` first and round-robins the three bulk -/// classes, so every pipeline keeps making progress. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -enum MsgClass { - /// Small/control messages (mempool, tx, ping, chainlock/islock `getdata`…): - /// always first, they are few and something is usually blocked on them. - Other, - Headers, - CfHeaders, - CFilters, - /// Block `getdata` — the wallet's matched blocks. - Blocks, -} - -fn classify(msg: &NetworkMessage) -> MsgClass { - match msg { - NetworkMessage::GetHeaders(_) | NetworkMessage::GetHeaders2(_) => MsgClass::Headers, - NetworkMessage::GetCFHeaders(_) => MsgClass::CfHeaders, - NetworkMessage::GetCFilters(_) => MsgClass::CFilters, - NetworkMessage::GetData(inv) - if !inv.is_empty() && inv.iter().all(|i| matches!(i, Inventory::Block(_))) => - { - MsgClass::Blocks - } - _ => MsgClass::Other, - } -} - -/// One queue per class, each behind its own lock: a pipeline enqueuing a burst -/// only contends with itself, never with the others. -struct MsgQueue { - other: Mutex>, - headers: Mutex>, - cfheaders: Mutex>, - cfilters: Mutex>, - blocks: Mutex>, - - /// Round-robin cursor over the three bulk classes. - turn: AtomicUsize, - len: AtomicUsize, - notify: Notify, -} - -struct State {} - -impl PeerNetworkManager { - pub async fn new(config: &ClientConfig) -> Self { - let mut discoverer = PeerDiscoverer::new(config); - let max_peers = config.max_peers.max(1) as usize; - - let connected_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); - let other_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); - let msg_queue = Arc::new(MsgQueue::new()); - - let (inbound_tx, inbound_rx) = mpsc::unbounded_channel(); - let subscribers: Subscribers = Arc::new(Mutex::new(HashMap::new())); - // Sized generously: `RequestOnFlight` is emitted per sent request, so the - // network-event bus is higher-volume than the peer-state events alone. - let (events_tx, _) = broadcast::channel(4096); - let shutdown = CancellationToken::new(); - // Total bytes read from all peers (download-only) and the global in-flight - // budget. The budget is NOT a fixed number: it starts at a tiny bootstrap - // just large enough to begin measuring, then `spawn_bandwidth_controller` - // sizes it from the measured download capacity (Little's Law). - let bytes = Arc::new(AtomicU64::new(0)); - let global_cap = Arc::new(AtomicUsize::new(max_peers.saturating_mul(4).max(8))); - - let best_tip = probe_and_select( - &mut discoverer, - &connected_peers, - &other_peers, - max_peers, - &inbound_tx, - &shutdown, - &bytes, - ) - .await; - - let pump = spawn_pump( - inbound_rx, - subscribers.clone(), - connected_peers.clone(), - events_tx.clone(), - msg_queue.clone(), - shutdown.clone(), - ); - - let router = spawn_router( - msg_queue.clone(), - connected_peers.clone(), - other_peers.clone(), - inbound_tx.clone(), - shutdown.clone(), - bytes.clone(), - global_cap.clone(), - events_tx.clone(), - ); - - spawn_bandwidth_controller( - bytes.clone(), - global_cap.clone(), - connected_peers.clone(), - shutdown.clone(), - ); - - let _ = events_tx.send(NetworkEvent::PeersUpdated); - - PeerNetworkManager { - connected_peers, - other_peers, - banned_peers: Vec::with_capacity(30), - discoverer, - msg_queue, - router, - pump, - inbound_tx, - subscribers, - events_tx, - best_tip, - max_peers, - shutdown, - } - } - - /// Tear down the network layer: stop the router and pump, and cancel every - /// peer reader so no more messages arrive. Called on client shutdown. - pub fn stop(&self) { - tracing::info!(target: "dash_spv::network2", "network manager stopping: cancelling tasks and peers"); - self.shutdown.cancel(); - } - - pub async fn send(&self, msg: NetworkMessage) { - self.msg_queue.push(msg).await; - } - - /// Note that `n` streaming requests served by `peer` have fully completed - /// (e.g. a `getcfilters` batch whose last `cfilter` just arrived), freeing - /// that peer's in-flight units and waking the router. Single-response - /// requests are freed in the peer's own reader instead. - pub async fn request_completed(&self, peer: SocketAddr, n: usize) { - if n == 0 { - return; - } - if let Some((p, _)) = - self.connected_peers.lock().await.iter().find(|(p, _)| p.addr() == peer) - { - p.response_completed(n); - } - self.msg_queue.notify.notify_one(); - } - - pub fn broadcast(&self, msg: NetworkMessage) { - let peers = self.connected_peers.clone(); - tokio::spawn(async move { - let guard = peers.lock().await; - for (peer, _) in guard.iter() { - let _ = peer.send(&msg).await; - } - }); - } - - /// Inject a message into the local pump as if it arrived from a peer, so - /// managers process it through the same path. Uses the `0.0.0.0:0` sentinel - /// address that managers treat as locally-originated. - pub async fn dispatch_local(&self, msg: NetworkMessage) { - let local: SocketAddr = ([0, 0, 0, 0], 0).into(); - let _ = self.inbound_tx.send(PeerEvent::Message(local, msg)); - } - - pub async fn subscribe(&self, kinds: &[MessageType]) -> UnboundedReceiver { - let (tx, rx) = mpsc::unbounded_channel(); - let mut subscribers = self.subscribers.lock().await; - for kind in kinds { - subscribers.entry(*kind).or_default().push(tx.clone()); - } - rx - } - - pub fn tip(&self) -> u32 { - self.best_tip - } - - pub fn events(&self) -> broadcast::Receiver { - self.events_tx.subscribe() - } -} - -fn spawn_router( - queue: Arc, - connected: Arc>>, - others: Arc>>, - inbound: UnboundedSender, - shutdown: CancellationToken, - bytes: Arc, - global_cap: Arc, - events: broadcast::Sender, -) -> JoinHandle<()> { - tokio::spawn(async move { - loop { - if shutdown.is_cancelled() { - break; - } - // Wait for work. `notify` fires both when a message is queued and - // when a response frees a peer slot. - if queue.len() == 0 { - tokio::select! { - _ = shutdown.cancelled() => break, - _ = queue.notify.notified() => continue, - } - } - - // Auto-connect more peers while the send queue is backed up: a deep - // queue means the connected peers can't serve our requests as fast as - // the pipelines produce them, so recruiting more raises the aggregate - // serving capacity and drains it — pushing the bottleneck toward OUR - // download link. Growth is bounded by `MAX_CONNECTED_PEERS`. Safe to - // grow past the initial `max_peers` now that each peer self-throttles - // to its MEASURED cap (a slow newcomer gets a tiny cap instead of - // feeding the old over-commit spiral). - if queue.len() > HIGH_DEMAND_QUEUE && connected.lock().await.len() < MAX_CONNECTED_PEERS - { - add_peers( - &connected, - &others, - &inbound, - &shutdown, - &bytes, - ADD_PEERS_BATCH, - MAX_CONNECTED_PEERS, - ) - .await; - } - - let peers = connected.lock().await; - let sent = - route_tick(&queue, &peers, global_cap.load(Ordering::Relaxed), &events).await; - drop(peers); - - if sent == 0 { - // Queue non-empty but every peer is at its in-flight cap: wait for - // a response to free capacity (the pump notifies on each response) - // — but never wait forever, because some of those "in-flight" - // requests may be dead. A peer that stops answering keeps its - // in-flight slots raised (the pipeline times the request out and - // re-queues it, but the peer is never told), so with enough dead - // requests every peer looks full, no response is coming, and the - // router would sleep on a notify that never fires — a hard sync - // stall with a backed-up queue. On the timeout we reclaim those - // slots and drop the peers that leaked them: they have had far - // longer than the pipelines' own timeout to answer, and leaving - // them connected would just route the freed work straight back to - // the peer that ignored it (the router picks the emptiest peer). - tokio::select! { - _ = shutdown.cancelled() => break, - _ = queue.notify.notified() => {}, - _ = tokio::time::sleep(STALL_CHECK) => { - let mut peers = connected.lock().await; - let before = peers.len(); - let mut reclaimed = 0; - peers.retain(|(p, _)| { - let n = p.reap_stale(STALE_REQUEST); - if n == 0 { - return true; - } - reclaimed += n; - // Drop the peer only if it has NEVER answered anything — - // that one is dead weight, and leaving it connected would - // route the freed work straight back to it (the router - // picks the emptiest peer). A peer that HAS served us is - // merely slow: a 2MB block over a weak link legitimately - // takes a while, and dropping those cost us 14 of 16 - // peers mid-sync. We only wanted the slot back. - let (completed, _) = p.latency_totals(); - if completed == 0 { - p.close(); - false - } else { - true - } - }); - if reclaimed > 0 { - tracing::warn!( - target: "dash_spv::network2", - "reclaimed {} stalled request slot(s); dropped {} never-responding peer(s) -> {} connected", - reclaimed, - before - peers.len(), - peers.len(), - ); - } - } - } - } - } - }) -} - -/// Extract the pipeline key of a router-paced request, so the router can report -/// it as on-flight. Returns `None` for non-pipeline messages. -fn request_keys(msg: &NetworkMessage) -> Vec { - match msg { - NetworkMessage::GetHeaders(m) | NetworkMessage::GetHeaders2(m) => { - m.locator_hashes.first().map(|h| RequestKey::Headers(*h)).into_iter().collect() - } - NetworkMessage::GetCFHeaders(m) => vec![RequestKey::CfHeaders(m.stop_hash)], - NetworkMessage::GetCFilters(m) => vec![RequestKey::CFilters(m.start_height)], - NetworkMessage::GetMnListD(m) => vec![RequestKey::MnListDiff(m.block_hash)], - // One `getdata` may name several blocks; each is its own tracked request. - NetworkMessage::GetData(inv) => inv - .iter() - .filter_map(|i| match i { - Inventory::Block(h) => Some(RequestKey::Block(*h)), - _ => None, - }) - .collect(), - _ => Vec::new(), - } -} - -async fn route_tick( - queue: &MsgQueue, - peers: &[(ConnectedPeer, State)], - global_cap: usize, - events: &broadcast::Sender, -) -> usize { - if peers.is_empty() { - return 0; - } - - // Free capacity this round = min(sum of per-peer room, global room). Each - // peer's cap is its MEASURED serving capacity (its bandwidth-delay product), - // sized by the controller from that peer's own completion rate and service - // time — fast peers carry more, slow peers less, with no fixed constant. The - // global cap is our measured download capacity. Whichever binds first limits - // this round, so we ride each peer's real ceiling without over-committing. - let total_in_flight: usize = peers.iter().map(|(p, _)| p.in_flight()).sum(); - let per_peer_room: usize = - peers.iter().map(|(p, _)| p.cap().saturating_sub(p.in_flight())).sum(); - let global_room = global_cap.saturating_sub(total_in_flight); - let capacity = per_peer_room.min(global_room); - if capacity == 0 { - return 0; - } - - let msgs = queue.pop_n(capacity).await; - let mut sent = 0; - // Anything popped that we could not put on the wire goes BACK on the queue. - // Dropping it would strand the owning pipeline forever: it has already marked - // the request as handed to the network, and its response timeout only starts - // when the router reports the request on the wire, so a dropped message is - // never re-sent and never times out. - let mut unsent: Vec = Vec::new(); - let mut msgs = msgs.into_iter(); - for msg in msgs.by_ref() { - // Send to the peer with the most free measured capacity. - let Some((peer, _)) = peers - .iter() - .filter(|(p, _)| p.in_flight() < p.cap()) - .max_by_key(|(p, _)| p.cap().saturating_sub(p.in_flight())) - else { - unsent.push(msg); // every peer is at its measured cap - break; - }; - if peer.send(&msg).await.is_ok() { - sent += 1; - // Tell the owning pipeline this request is now on the wire so it can - // start the response timeout from here, not from when it was queued. - for key in request_keys(&msg) { - let _ = events.send(NetworkEvent::RequestOnFlight(key)); - } - } else { - tracing::warn!(target: "dash_spv::network2", "router: send to {} failed", peer.addr()); - unsent.push(msg); - } - } - unsent.extend(msgs); // whatever the loop never reached - queue.push_front_all(unsent).await; - - if sent > 0 { - tracing::debug!( - target: "dash_spv::network2", - "router: sent {} | peers={} queue={}", - sent, - peers.len(), - queue.len(), - ); - } - sent -} - -/// Sizes the global in-flight budget from MEASURED download capacity — no fixed -/// magic number, per the design goal of estimating how many requests the current -/// network can absorb before it saturates. -/// -/// The estimate is Little's Law applied to downloads: the number of requests in -/// flight that sustains a completion rate `λ` at an uncongested per-request -/// service time `W` is `L = λ · W`. We measure both from the peers' response -/// stream — `λ` = requests completed per second, `W` = average time from send to -/// the response that completes the request (for `getcfilters`, dominated by the -/// download time of its ~1000 `cfilter`s, i.e. our downlink). We track the -/// minimum `W` as the uncongested baseline (the pipe's true latency at the front -/// of the knee) and target `L = λ · min_W`, probing slightly past it. -/// -/// Saturation is detected by `W` INFLATION, not by throughput: once in-flight -/// exceeds the bandwidth-delay product, extra requests just queue at the peers, -/// so `W` climbs while `λ` (and the download rate) plateaus. That inflation is -/// visible even though a raw throughput meter can't see the knee (a prior -/// throughput hill-climb ran away and regressed sync ~8x because the backlog -/// kept bytes flowing). When `W > min_W · INFLATE` we stop probing and shrink -/// back toward the sustaining level, so we ride just below saturation. -fn spawn_bandwidth_controller( - bytes: Arc, - cap: Arc, - connected: Arc>>, - shutdown: CancellationToken, -) -> JoinHandle<()> { - const WINDOW: Duration = Duration::from_millis(500); - const FLOOR_PER_PEER: usize = 2; // global floor = peers · this - const PEER_CEIL: usize = 32; // per-connection sanity bound on in-flight - const RISE: f64 = 1.05; // throughput must climb 5% to justify a bigger cap - const DROP: f64 = 0.85; // throughput below this·last => over-commit, back off - const REPROBE: u32 = 8; // plateau windows to hold before nudging the cap up - const IDLE_BPS: f64 = 1.0e6; // downlink under 1 MB/s = idle, hold the cap - const EMA_ALPHA: f64 = 0.5; // smoothing for the noisy per-window rate - let window_s = WINDOW.as_secs_f64(); - - tokio::spawn(async move { - let mut last_bytes = bytes.load(Ordering::Relaxed); - let mut rate_ema = 0.0f64; // smoothed downlink bytes/s - let mut last_rate = 0.0f64; // smoothed rate at the previous cap adjustment - let mut hold = 0u32; // consecutive plateau windows - let mut ticker = tokio::time::interval(WINDOW); - loop { - tokio::select! { - _ = shutdown.cancelled() => break, - _ = ticker.tick() => {} - } - - // Downlink throughput (bytes read off the sockets — our downlink only). - // This is the saturation signal: unlike per-request service time — which - // balloons with cfilter payload size and out-of-order batch completion, - // reading "saturated" forever and pinning the cap at its floor — bytes/s - // directly reflects whether we are using the pipe. We grow the in-flight - // budget while throughput keeps climbing with it and stop at the plateau - // (the bandwidth-delay product: past it, more in-flight only grows queues, - // not bytes/s), backing off if it collapses (peers over-committed). - let now_bytes = bytes.load(Ordering::Relaxed); - let dl_rate = now_bytes.saturating_sub(last_bytes) as f64 / window_s; - last_bytes = now_bytes; - rate_ema = if rate_ema == 0.0 { - dl_rate - } else { - EMA_ALPHA * dl_rate + (1.0 - EMA_ALPHA) * rate_ema - }; - - let npeers = connected.lock().await.len(); - if npeers == 0 { - continue; - } - let floor = (npeers * FLOOR_PER_PEER).max(FLOOR_PER_PEER); - let ceiling = (npeers * PEER_CEIL).max(floor + 1); - let step = npeers.max(4); // ~one extra slot per peer per window - let cur = cap.load(Ordering::Relaxed); - - // Gradient hill-climb on smoothed throughput. - let (new, action) = if rate_ema < IDLE_BPS { - // Nothing meaningful downloading (e.g. the commit tail): hold the - // budget steady so it is ready when the download resumes. - (cur, "idle") - } else if cur <= floor || rate_ema >= last_rate * RISE { - // Still gaining (or at the floor): push the budget up. - hold = 0; - ((cur + step).min(ceiling), "grow") - } else if rate_ema < last_rate * DROP { - // Throughput collapsed — the peers are over-committed. Back off. - hold = 0; - (((cur as f64 * 0.8) as usize).max(floor), "backoff") - } else { - // Plateau: we are at the knee. Hold, re-probing up occasionally to - // catch a capacity increase (a faster peer, less congestion). - hold += 1; - if hold >= REPROBE { - hold = 0; - ((cur + step).min(ceiling), "reprobe") - } else { - (cur, "hold") - } - }; - // Anchor RISE/DROP to the rate at each real adjustment (skip idle/hold - // windows) so the next comparison is like-for-like. - if matches!(action, "grow" | "backoff" | "reprobe") { - last_rate = rate_ema; - } - cap.store(new, Ordering::Relaxed); - - // Spread the global budget evenly across peers with a slot of headroom, - // so the GLOBAL cap is the binding constraint (route_tick fills the - // emptiest peer first, so fast peers naturally serve more) while no - // single connection is over-committed beyond PEER_CEIL. - let per_peer = (new.div_ceil(npeers) + 1).clamp(FLOOR_PER_PEER, PEER_CEIL); - { - let g = connected.lock().await; - for (p, _) in g.iter() { - p.set_cap(per_peer); - } - } - - tracing::debug!( - target: "dash_spv::network2", - "bandwidth: {:.1} MB/s (ema {:.1}) | {} | global cap={} | peers={} per-peer cap={}", - dl_rate / 1e6, - rate_ema / 1e6, - action, - new, - npeers, - per_peer, - ); - } - }) -} - -async fn add_peers( - connected: &Mutex>, - others: &Mutex>, - inbound: &UnboundedSender, - shutdown: &CancellationToken, - bytes: &Arc, - batch: usize, - max_peers: usize, -) { - // Never grow past the operating cap; only backfill the deficit toward it. - let deficit = max_peers.saturating_sub(connected.lock().await.len()); - if deficit == 0 { - return; - } - let candidates: Vec = { - let mut others = others.lock().await; - let take = batch.min(deficit).min(others.len()); - others.drain(..take).collect() - }; - - let mut added = 0; - for candidate in candidates { - if let Ok(peer) = candidate.connect(inbound.clone(), shutdown.clone(), bytes.clone()).await - { - connected.lock().await.push((peer, State {})); - added += 1; - } - } - - if added > 0 { - let total = connected.lock().await.len(); - tracing::info!( - target: "dash_spv::network2", - "high demand: added {} peers -> {} connected", - added, - total, - ); - } -} - -async fn probe_and_select( - discoverer: &mut PeerDiscoverer, - connected: &Mutex>, - others: &Mutex>, - max_peers: usize, - inbound: &UnboundedSender, - shutdown: &CancellationToken, - bytes: &Arc, -) -> u32 { - const CONNECT_CHUNK: usize = 16; // bounded concurrent handshakes per round - const FAST_LAG_MS: u32 = 100; // prefer peers that ping under this - - // Sort key for latency: lower is better; treat an unmeasured lag (0) as worst. - fn lag_key(p: &ConnectedPeer) -> u32 { - match p.lag_ms() { - 0 => u32::MAX, - ms => ms, - } - } - - let mut candidates = discoverer.get(PROBE_BATCH).await; - // Cap how many candidates we handshake before settling. This bounds startup: - // real peers often ping over 100ms, so without a limit the "wait for fast - // peers" loop would probe the entire batch. We try a few times `max_peers` - // and then take the lowest-latency of whatever connected. - let attempt_max = (max_peers * 3).max(CONNECT_CHUNK * 2); - let mut attempted = 0usize; - let mut kept: Vec = Vec::new(); - let mut fallback: Vec = Vec::new(); // connected, not fast enough - let mut tip = 0u32; - - // Connect a chunk at a time and STOP as soon as we have `max_peers` fast - // peers (or we've tried enough), so sync starts without waiting on the whole - // probe batch. We keep only the peers we'll actually query connected; - // everything else stays a bare address for on-demand recruitment. - while kept.len() < max_peers && !candidates.is_empty() && attempted < attempt_max { - let take = CONNECT_CHUNK.min(candidates.len()); - attempted += take; - let batch: Vec = candidates.drain(..take).collect(); - let results = join_all( - batch.into_iter().map(|c| c.connect(inbound.clone(), shutdown.clone(), bytes.clone())), - ) - .await; - for peer in results.into_iter().filter_map(Result::ok) { - tip = tip.max(peer.version().start_height.max(0) as u32); - let lag = peer.lag_ms(); - if kept.len() < max_peers && lag > 0 && lag < FAST_LAG_MS { - kept.push(peer); - } else { - fallback.push(peer); - } - } - } - - // If too few peers were fast, fill up to `max_peers` with the lowest-latency - // fallbacks; close and remember the rest as backups. - fallback.sort_by_key(lag_key); - while kept.len() < max_peers && !fallback.is_empty() { - kept.push(fallback.remove(0)); - } - let mut backups: Vec = candidates; // un-probed addresses - for peer in fallback { - peer.close(); // drop the connection we won't use - backups.push(peer.disconnect()); - } - - let fast = kept.iter().filter(|p| p.lag_ms() > 0 && p.lag_ms() < FAST_LAG_MS).count(); - tracing::info!( - target: "dash_spv::network2", - "probe: connected {} peers ({} fast <{}ms), {} backups | tip={}", - kept.len(), - fast, - FAST_LAG_MS, - backups.len(), - tip, - ); - - *connected.lock().await = kept.into_iter().map(|peer| (peer, State {})).collect(); - *others.lock().await = backups; - - tip -} - -fn spawn_pump( - mut inbound: UnboundedReceiver, - subscribers: Subscribers, - connected: Arc>>, - events: broadcast::Sender, - queue: Arc, - shutdown: CancellationToken, -) -> JoinHandle<()> { - tokio::spawn(async move { - // Per-peer received-message counter to check load balance across peers. - let mut recv_by_peer: HashMap = HashMap::new(); - let mut total_recv: u64 = 0; - loop { - let t_idle = std::time::Instant::now(); - let event = tokio::select! { - _ = shutdown.cancelled() => break, - ev = inbound.recv() => match ev { - Some(ev) => ev, - None => break, - }, - }; - crate::timer::P_PUMP_IDLE.add(t_idle.elapsed()); - match event { - PeerEvent::Message(addr, msg) => { - let t_busy = std::time::Instant::now(); - *recv_by_peer.entry(addr).or_insert(0) += 1; - total_recv += 1; - if total_recv % 250_000 == 0 { - let mut dist: Vec<(SocketAddr, u64)> = - recv_by_peer.iter().map(|(a, c)| (*a, *c)).collect(); - dist.sort_by_key(|(_, c)| std::cmp::Reverse(*c)); - tracing::info!( - target: "filt_depth", - "[t+{}ms] recv balance ({} peers, {} total): {:?}", - crate::timer::elapsed_ms(), - dist.len(), - total_recv, - dist, - ); - // Per-peer request->response latency (avg / worst). - let lat: Vec<(SocketAddr, u64, String)> = connected - .lock() - .await - .iter() - .map(|(p, _)| { - let (n, avg, max) = p.latency_stats(); - (p.addr(), n, format!("avg={avg:.1}ms max={max:.1}ms")) - }) - .collect(); - tracing::info!( - target: "filt_depth", - "[t+{}ms] peer latency: {:?}", - crate::timer::elapsed_ms(), - lat, - ); - } - let mt = MessageType::from_cmd(msg.cmd()); - // Single-message responses free a slot in the peer's reader; - // wake the router so it can use the freed capacity. `cfilter` - // is skipped — its batch frees a slot via `request_completed`, - // which notifies once per ~1000 messages instead of each. - if mt != Some(MessageType::CFilter) { - queue.notify.notify_one(); - } - - { - let mut subscribers = subscribers.lock().await; - if let Some(list) = mt.and_then(|t| subscribers.get_mut(&t)) { - let last = list.len().saturating_sub(1); - let mut msg = Some(Arc::new(msg)); - let mut idx = 0; - - list.retain(|tx| { - // Hand the *last* subscriber our own reference instead of a - // clone. Every heavy message type (block, cfilter, cfheaders, - // headers) has exactly one subscriber, so it arrives with a - // refcount of 1 and the manager can take the payload without - // copying it. Only `inv`, which is tiny, is really shared. - let shared = if idx == last { - msg.take().expect("taken once, on the final subscriber") - } else { - Arc::clone( - msg.as_ref().expect("held until the final subscriber"), - ) - }; - idx += 1; - - tx.send((addr, shared)).is_ok() - }); - } - } - crate::timer::P_PUMP_BUSY.add(t_busy.elapsed()); - } - PeerEvent::Disconnected(addr) => { - let remaining = { - let mut guard = connected.lock().await; - guard.retain(|(peer, _)| peer.addr() != addr); - guard.len() - }; - tracing::info!( - target: "dash_spv::network2", - "peer disconnected: {} | {} peers remaining", - addr, - remaining, - ); - let _ = events.send(NetworkEvent::PeerDisconnected(addr)); - } - } - } - }) -} - -impl MsgQueue { - fn new() -> Self { - Self { - other: Mutex::new(VecDeque::with_capacity(30)), - headers: Mutex::new(VecDeque::with_capacity(30)), - cfheaders: Mutex::new(VecDeque::with_capacity(30)), - cfilters: Mutex::new(VecDeque::with_capacity(30)), - blocks: Mutex::new(VecDeque::with_capacity(30)), - turn: AtomicUsize::new(0), - len: AtomicUsize::new(0), - notify: Notify::new(), - } - } - - fn len(&self) -> usize { - self.len.load(Ordering::SeqCst) - } - - fn queue(&self, class: MsgClass) -> &Mutex> { - match class { - MsgClass::Other => &self.other, - MsgClass::Headers => &self.headers, - MsgClass::CfHeaders => &self.cfheaders, - MsgClass::CFilters => &self.cfilters, - MsgClass::Blocks => &self.blocks, - } - } - - /// Take up to `n` messages in scheduling order: all pending control traffic - /// first (it is small, and something is usually blocked on it), then the bulk - /// classes by WEIGHTED rotation. - /// - /// Filters get most of the turns because they are the phase that actually costs - /// time: a compact filter is ~700 bytes per block against 32 for a filter - /// header, so the filter download is what the sync waits on. - /// - /// But not ALL the turns — strict priority would be a trap. Filters can only be - /// verified (and therefore stored, scanned and committed) once the filter-header - /// chain has been verified through their range, so starving cfheaders behind a - /// deep cfilters backlog stalls the very phase we were trying to favour. The - /// weights keep the small, cheap requests flowing while filters take the rest. - async fn pop_n(&self, n: usize) -> Vec { - if n == 0 { - return Vec::new(); - } - let mut out: Vec = Vec::with_capacity(n); - - { - let mut other = self.other.lock().await; - let take = n.min(other.len()); - out.extend(other.drain(..take)); - } - - // One lap of the rotation: filters four turns for every one of each header - // class. A class with an empty queue simply yields its turn. - // Filters take half the lap (they are the bytes), blocks a quarter (they - // gate the gap-limit cascade, so stalling them stretches the tail), and the - // two small header classes an eighth each — enough to keep the chains that - // unblock the filters moving. - const ROTATION: [MsgClass; 8] = [ - MsgClass::CFilters, - MsgClass::Blocks, - MsgClass::CFilters, - MsgClass::Headers, - MsgClass::CFilters, - MsgClass::Blocks, - MsgClass::CFilters, - MsgClass::CfHeaders, - ]; - let mut turn = self.turn.load(Ordering::Relaxed); - while out.len() < n { - let mut got = false; - for _ in 0..ROTATION.len() { - if out.len() == n { - break; - } - let class = ROTATION[turn % ROTATION.len()]; - turn = turn.wrapping_add(1); - if let Some(msg) = self.queue(class).lock().await.pop_front() { - out.push(msg); - got = true; - } - } - if !got { - break; // every bulk class is empty - } - } - self.turn.store(turn, Ordering::Relaxed); - - self.len.fetch_sub(out.len(), Ordering::SeqCst); - out - } - - async fn push(&self, msg: NetworkMessage) { - self.queue(classify(&msg)).lock().await.push_back(msg); - self.len.fetch_add(1, Ordering::SeqCst); - self.notify.notify_one(); - } - - /// Return messages that were popped but could not be sent to the FRONT of - /// their own class queue, preserving order. A popped message must never be - /// dropped: the owning pipeline has already recorded it as handed to the - /// network and only starts its response timeout once the router reports it on - /// the wire, so a dropped message is one the pipeline waits on forever — a - /// permanent sync stall (observed as: queue backed up, 0 MB/s, no sends, no - /// timeouts). - async fn push_front_all(&self, msgs: Vec) { - if msgs.is_empty() { - return; - } - let n = msgs.len(); - for msg in msgs.into_iter().rev() { - self.queue(classify(&msg)).lock().await.push_front(msg); - } - self.len.fetch_add(n, Ordering::SeqCst); - self.notify.notify_one(); - } -} diff --git a/dash-spv/src/network2/mod.rs b/dash-spv/src/network2/mod.rs deleted file mode 100644 index 15bedd101..000000000 --- a/dash-spv/src/network2/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod discovery; -mod manager; -mod peer; - -pub use manager::{MessageType, NetworkEvent, PeerNetworkManager, RequestKey}; diff --git a/dash-spv/src/network2/peer.rs b/dash-spv/src/network2/peer.rs deleted file mode 100644 index 6cb66f524..000000000 --- a/dash-spv/src/network2/peer.rs +++ /dev/null @@ -1,536 +0,0 @@ -use std::collections::VecDeque; -use std::net::SocketAddr; -use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex as StdMutex}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - -/// Per-peer response-latency tracker: the time each pipeline request spends in -/// flight, from send to the response that completes it. Send times are queued -/// FIFO; each completing response pops the oldest and folds the elapsed time -/// into running total/count/max (so we can report avg and worst per peer). -#[derive(Default)] -pub(crate) struct Latency { - pending: StdMutex>, - total_ns: AtomicU64, - count: AtomicU64, - max_ns: AtomicU64, -} - -impl Latency { - fn on_send(&self) { - self.pending.lock().unwrap().push_back(Instant::now()); - } - - /// Drop pending sends older than `timeout` (the pipelines have long since - /// timed them out and re-queued them elsewhere), returning how many. Their - /// in-flight slots would otherwise be held forever — see - /// [`ConnectedPeer::reap_stale`]. - fn reap(&self, timeout: Duration) -> usize { - let mut pending = self.pending.lock().unwrap(); - let now = Instant::now(); - let mut n = 0; - while let Some(&front) = pending.front() { - if now.duration_since(front) > timeout { - pending.pop_front(); - n += 1; - } else { - break; // FIFO: the rest are younger - } - } - n - } - - /// Pop the oldest pending send and record its round-trip. - fn complete_one(&self) { - let sent = self.pending.lock().unwrap().pop_front(); - if let Some(sent) = sent { - let ns = sent.elapsed().as_nanos() as u64; - self.total_ns.fetch_add(ns, Ordering::Relaxed); - self.count.fetch_add(1, Ordering::Relaxed); - self.max_ns.fetch_max(ns, Ordering::Relaxed); - } - } - - /// Cumulative (completed request count, total service-time nanoseconds). - /// Deltas between two reads give the windowed completion rate and average - /// service time used to size the in-flight budget by Little's Law. - fn totals(&self) -> (u64, u64) { - (self.count.load(Ordering::Relaxed), self.total_ns.load(Ordering::Relaxed)) - } - - /// (completed request count, average ms, worst ms). - fn snapshot(&self) -> (u64, f64, f64) { - let count = self.count.load(Ordering::Relaxed); - let total = self.total_ns.load(Ordering::Relaxed); - let max = self.max_ns.load(Ordering::Relaxed); - let avg_ms = if count > 0 { - total as f64 / count as f64 / 1e6 - } else { - 0.0 - }; - (count, avg_ms, max as f64 / 1e6) - } -} - -use dashcore::{ - consensus::encode, - network::{ - address::Address, - constants::ServiceFlags, - message::{NetworkMessage, RawNetworkMessage, RawNetworkMessageCodec}, - message_network::VersionMessage, - }, - Network, -}; -use futures::lock::Mutex; -use tokio::sync::mpsc::UnboundedSender; -use tokio::{ - io::{AsyncRead, AsyncWriteExt, ReadBuf}, - net::{ - tcp::{OwnedReadHalf, OwnedWriteHalf}, - TcpStream, - }, -}; -use tokio_stream::StreamExt; -use tokio_util::codec::FramedRead; -use tokio_util::sync::CancellationToken; - -use crate::{error::NetworkResult, NetworkError}; - -const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); -const USER_AGENT: &str = concat!("/dash-spv:", env!("CARGO_PKG_VERSION"), "/"); - -/// Wraps a socket read half and adds every byte read into a shared counter, so -/// the network manager can estimate download throughput (and size the global -/// in-flight budget to ~90% of it). -struct CountingReader { - inner: R, - bytes: Arc, -} - -impl AsyncRead for CountingReader { - fn poll_read( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> std::task::Poll> { - let before = buf.filled().len(); - let r = std::pin::Pin::new(&mut self.inner).poll_read(cx, buf); - if let std::task::Poll::Ready(Ok(())) = &r { - let n = buf.filled().len() - before; - self.bytes.fetch_add(n as u64, Ordering::Relaxed); - } - r - } -} - -type PeerReader = FramedRead, RawNetworkMessageCodec>; - -pub enum PeerEvent { - Message(SocketAddr, NetworkMessage), - Disconnected(SocketAddr), -} - -pub struct ConnectedPeer { - network: Network, - addr: SocketAddr, - version: VersionMessage, - lag_ms: AtomicU32, - in_flight: Arc, - latency: Arc, - writer: Arc>, - /// This peer's measured serving capacity — the number of requests it can have - /// in flight before ITS responses start queuing (its bandwidth-delay product). - /// Sized continuously by the bandwidth controller from the peer's own - /// completion rate and service time, mirroring how the global budget is sized - /// from our download rate. Fast peers earn a high cap, slow peers a low one. - cap: Arc, - /// Per-connection cancel token (child of the global shutdown). Cancelling it - /// stops this peer's reader and closes the socket. Used to drop peers we - /// probed but don't keep, so we only hold connections we actually use. - token: CancellationToken, -} - -pub struct DisconnectedPeer { - network: Network, - addr: SocketAddr, -} - -impl ConnectedPeer { - pub fn addr(&self) -> SocketAddr { - self.addr - } - - pub fn version(&self) -> &VersionMessage { - &self.version - } - - /// Net in-flight to this peer: `+1` per message we send it, `-1` per message - /// we read from it (managed internally by `send` and the reader task). The - /// router reads this to send to the least-loaded peer. - pub fn in_flight(&self) -> usize { - self.in_flight.load(Ordering::Relaxed) - } - - pub fn disconnect(self) -> DisconnectedPeer { - DisconnectedPeer { - network: self.network, - addr: self.addr, - } - } - - pub async fn send(&self, msg: &NetworkMessage) -> NetworkResult<()> { - // TODO: Take a reference to msg instead of cloning it - let raw = RawNetworkMessage { - magic: self.network.magic(), - payload: msg.clone(), - }; - let serialized = encode::serialize(&raw); - - if let Err(e) = self.writer.lock().await.write_all(&serialized).await { - tracing::warn!("Disconnecting {} due to write error: {}", self.addr, e); - return Err(NetworkError::ConnectionFailed(format!("Write failed: {}", e))); - } - // A pipeline request counts as one unit of in-flight work for this peer. - if is_pipeline_request(msg) { - self.in_flight.fetch_add(1, Ordering::Relaxed); - self.latency.on_send(); - } - Ok(()) - } - - /// Note that `n` earlier pipeline requests have fully completed, freeing that - /// much in-flight work. Used for streaming responses (`getcfilters` -> many - /// `cfilter`s) that the reader can't attribute to a finished request on its - /// own; single-response requests are decremented directly in the reader. - pub(crate) fn response_completed(&self, n: usize) { - let _ = self - .in_flight - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(n))); - for _ in 0..n { - self.latency.complete_one(); - } - } - - /// Reclaim the in-flight slots of requests this peer has not answered within - /// `timeout`, returning how many. Without this they leak: when a pipeline - /// times a request out it re-queues it, but nothing tells the peer its - /// request died, so the peer's `in_flight` counter stays raised forever. - /// Enough dead requests and EVERY peer sits at its cap with nothing actually - /// on the wire — the router sees zero free capacity, stops sending, and the - /// sync deadlocks with a full queue (reproduced at 32 peers: 0 MB/s, queue - /// 140, no sends). The count also identifies the peer as unresponsive, so the - /// caller can drop it rather than keep handing it work. - pub(crate) fn reap_stale(&self, timeout: Duration) -> usize { - let n = self.latency.reap(timeout); - if n > 0 { - let _ = self - .in_flight - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(n))); - } - n - } - - /// Per-peer response latency: (completed requests, average ms, worst ms). - pub(crate) fn latency_stats(&self) -> (u64, f64, f64) { - self.latency.snapshot() - } - - /// Cumulative (completed request count, total service-time ns) for this peer. - /// The bandwidth controller diffs these across a window to get the download - /// completion rate and service time that size the in-flight budget. - pub(crate) fn latency_totals(&self) -> (u64, u64) { - self.latency.totals() - } - - /// Handshake round-trip latency in ms (0 if unmeasured). - pub(crate) fn lag_ms(&self) -> u32 { - self.lag_ms.load(Ordering::Relaxed) - } - - /// Close this connection: cancel its reader so the socket shuts down. Used to - /// drop probed-but-unselected peers instead of leaking their readers. - pub(crate) fn close(&self) { - self.token.cancel(); - } - - /// This peer's current measured in-flight capacity (its serving BDP). - pub(crate) fn cap(&self) -> usize { - self.cap.load(Ordering::Relaxed) - } - - /// Update this peer's measured in-flight capacity (called by the controller). - pub(crate) fn set_cap(&self, n: usize) { - self.cap.store(n, Ordering::Relaxed); - } -} - -/// Pipeline requests we send and expect a response for (each adds one in-flight). -fn is_pipeline_request(msg: &NetworkMessage) -> bool { - match msg { - NetworkMessage::GetHeaders(_) - | NetworkMessage::GetHeaders2(_) - | NetworkMessage::GetCFHeaders(_) - | NetworkMessage::GetCFilters(_) => true, - // Block `getdata` is paced like any other request: the blocks pipeline - // sends ONE block per message, so a request is exactly one in-flight unit - // and one `block` in reply. Other `getdata` (chainlocks, islocks, txs) is - // control traffic — it carries several inventory items whose replies the - // reader cannot attribute one-for-one, so counting it would leak slots. - NetworkMessage::GetData(inv) => { - !inv.is_empty() - && inv - .iter() - .all(|i| matches!(i, dashcore::network::message_blockdata::Inventory::Block(_))) - } - _ => false, - } -} - -/// Single-message responses: the reader decrements one in-flight per message. -/// `cfilter` is excluded — one `getcfilters` yields up to 1000 `cfilter`s, so its -/// unit is freed once per batch by the filters pipeline via `response_completed`. -fn is_single_response(msg: &NetworkMessage) -> bool { - matches!( - msg, - NetworkMessage::Headers(_) - | NetworkMessage::Headers2(_) - | NetworkMessage::CFHeaders(_) - | NetworkMessage::Block(_) - ) -} - -impl DisconnectedPeer { - pub fn new(addr: SocketAddr, network: Network) -> Self { - DisconnectedPeer { - network, - addr, - } - } - - pub async fn connect( - self, - inbound: UnboundedSender, - shutdown: CancellationToken, - bytes: Arc, - ) -> NetworkResult { - let stream = TcpStream::connect(&self.addr).await.map_err(|e| { - NetworkError::ConnectionFailed(format!("Failed to connect to {}: {}", self.addr, e)) - })?; - - let (read_half, mut writer) = stream.into_split(); - let mut reader = FramedRead::new( - CountingReader { - inner: read_half, - bytes, - }, - RawNetworkMessageCodec, - ); - let magic = self.network.magic(); - - handshake_send(&mut writer, magic, NetworkMessage::Version(build_version(self.addr))) - .await?; - - let deadline = tokio::time::Instant::now() + HANDSHAKE_TIMEOUT; - let mut peer_version: Option = None; - let mut got_verack = false; - - while !(peer_version.is_some() && got_verack) { - let raw = match tokio::time::timeout_at(deadline, reader.next()).await { - Err(_) => return Err(NetworkError::Timeout), - Ok(None) => return Err(NetworkError::PeerDisconnected), - Ok(Some(Err(e))) => return Err(e.into()), - Ok(Some(Ok(raw))) => raw, - }; - if raw.magic != magic { - return Err(NetworkError::ProtocolError("wrong network magic".into())); - } - match raw.payload { - NetworkMessage::Version(v) => { - // BIP155: sendaddrv2 must be sent BEFORE verack. - handshake_send(&mut writer, magic, NetworkMessage::SendAddrV2).await?; - handshake_send(&mut writer, magic, NetworkMessage::Verack).await?; - peer_version = Some(v); - } - NetworkMessage::Verack => got_verack = true, - NetworkMessage::Ping(n) => { - handshake_send(&mut writer, magic, NetworkMessage::Pong(n)).await? - } - _ => {} - } - } - - let version = peer_version.ok_or(NetworkError::PeerDisconnected)?; - - // Announce sendheaders only after the handshake is fully complete. - handshake_send(&mut writer, magic, NetworkMessage::SendHeaders).await?; - - // Measure round-trip lag with a post-handshake ping/pong. Sending a ping - // before the handshake completes makes some peers drop us, so we do it here. - let mut lag_ms: u32 = 0; - let ping_nonce: u64 = rand::random(); - let ping_sent = tokio::time::Instant::now(); - if handshake_send(&mut writer, magic, NetworkMessage::Ping(ping_nonce)).await.is_ok() { - let deadline = ping_sent + HANDSHAKE_TIMEOUT; - while let Ok(Some(Ok(raw))) = tokio::time::timeout_at(deadline, reader.next()).await { - if raw.magic != magic { - continue; - } - match raw.payload { - NetworkMessage::Pong(n) if n == ping_nonce => { - lag_ms = ping_sent.elapsed().as_millis().clamp(1, u32::MAX as u128) as u32; - break; - } - NetworkMessage::Ping(n) => { - let _ = handshake_send(&mut writer, magic, NetworkMessage::Pong(n)).await; - } - _ => {} - } - } - } - - let writer = Arc::new(Mutex::new(writer)); - let in_flight = Arc::new(AtomicUsize::new(0)); - let latency = Arc::new(Latency::default()); - // Start with a tiny in-flight capacity; the controller grows it from this - // peer's measured serving rate. - let cap = Arc::new(AtomicUsize::new(2)); - // Per-connection token: cancelled by the global shutdown (parent) OR by - // `close()` to drop just this peer. - let token = shutdown.child_token(); - spawn_reader( - self.addr, - magic, - reader, - writer.clone(), - inbound, - in_flight.clone(), - latency.clone(), - token.clone(), - ); - - tracing::debug!( - target: "dash_spv::network2", - "peer connected: {} | lag={}ms height={}", - self.addr, - lag_ms, - version.start_height, - ); - - Ok(ConnectedPeer { - network: self.network, - addr: self.addr, - version, - lag_ms: AtomicU32::new(lag_ms), - in_flight, - latency, - writer, - cap, - token, - }) - } -} - -fn spawn_reader( - addr: SocketAddr, - magic: u32, - mut reader: PeerReader, - writer: Arc>, - inbound: UnboundedSender, - in_flight: Arc, - latency: Arc, - shutdown: CancellationToken, -) { - tokio::spawn(async move { - loop { - let t_recv = std::time::Instant::now(); - let next = tokio::select! { - _ = shutdown.cancelled() => break, - next = reader.next() => next, - }; - crate::timer::P_READ_RECV.add(t_recv.elapsed()); - match next { - None => break, - Some(Err(e)) => { - tracing::error!("NETWORK: reader {} stopped: {}", addr, e); - break; - } - Some(Ok(raw)) => { - if raw.magic != magic { - continue; - } - // A single-message response completes one unit of in-flight - // work. Streaming responses (cfilter) are freed per batch by - // the filters pipeline instead. - if is_single_response(&raw.payload) { - let _ = in_flight.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| { - Some(v.saturating_sub(1)) - }); - latency.complete_one(); - } - match raw.payload { - NetworkMessage::Ping(nonce) => { - let pong = RawNetworkMessage { - magic, - payload: NetworkMessage::Pong(nonce), - }; - if writer - .lock() - .await - .write_all(&encode::serialize(&pong)) - .await - .is_err() - { - break; - } - } - payload => { - if inbound.send(PeerEvent::Message(addr, payload)).is_err() { - break; - } - } - } - } - } - } - - tracing::info!("NETWORK: peer {} disconnected", addr); - let _ = inbound.send(PeerEvent::Disconnected(addr)); - }); -} - -fn build_version(peer: SocketAddr) -> VersionMessage { - let now = SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0); - let unspecified: SocketAddr = ([0u8, 0, 0, 0], 0).into(); - VersionMessage::new( - ServiceFlags::NONE, - now, - Address::new(&peer, ServiceFlags::NETWORK), - Address::new(&unspecified, ServiceFlags::NONE), - rand::random(), - USER_AGENT.to_string(), - 0, - false, - [0u8; 32], - ) -} - -async fn handshake_send( - writer: &mut OwnedWriteHalf, - magic: u32, - payload: NetworkMessage, -) -> NetworkResult<()> { - let raw = RawNetworkMessage { - magic, - payload, - }; - writer - .write_all(&encode::serialize(&raw)) - .await - .map_err(|e| NetworkError::ConnectionFailed(format!("handshake write failed: {}", e)))?; - writer - .flush() - .await - .map_err(|e| NetworkError::ConnectionFailed(format!("handshake flush failed: {}", e)))?; - Ok(()) -} diff --git a/dash-spv/src/storage/peers.rs b/dash-spv/src/storage/peers.rs index 360e83650..cdf818fe1 100644 --- a/dash-spv/src/storage/peers.rs +++ b/dash-spv/src/storage/peers.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fs::File, io::BufReader, net::SocketAddr, path::PathBuf}; +use std::{fs::File, io::BufReader, path::PathBuf}; use tokio::fs; @@ -10,7 +10,6 @@ use dashcore::{ use crate::{ error::StorageResult, - network::PeerReputation, storage::{io::atomic_write, PersistentStorage}, StorageError, }; @@ -23,13 +22,6 @@ pub trait PeerStorage { ) -> StorageResult<()>; async fn load_peers(&self) -> StorageResult>; - - async fn save_peers_reputation( - &self, - reputations: &HashMap, - ) -> StorageResult<()>; - - async fn load_peers_reputation(&self) -> StorageResult>; } pub struct PersistentPeerStorage { @@ -42,10 +34,6 @@ impl PersistentPeerStorage { fn peers_data_file(&self) -> PathBuf { self.storage_path.join("peers.dat") } - - fn peers_reputation_file(&self) -> PathBuf { - self.storage_path.join("reputations.json") - } } #[async_trait] @@ -123,38 +111,6 @@ impl PeerStorage for PersistentPeerStorage { Ok(peers) } - - async fn save_peers_reputation( - &self, - reputations: &HashMap, - ) -> StorageResult<()> { - let reputation_file = self.peers_reputation_file(); - - let json = serde_json::to_string_pretty(reputations).map_err(|e| { - StorageError::Serialization(format!("Failed to serialize peers reputations: {e}")) - })?; - - let reputation_file_parent = reputation_file - .parent() - .ok_or(StorageError::NotFound("reputation_file doesn't have a parent".to_string()))?; - - fs::create_dir_all(reputation_file_parent).await?; - - atomic_write(&reputation_file, json.as_bytes()).await - } - - async fn load_peers_reputation(&self) -> StorageResult> { - let reputation_file = self.peers_reputation_file(); - - if !fs::try_exists(&reputation_file).await? { - return Ok(HashMap::new()); - } - - let json = fs::read_to_string(reputation_file).await?; - serde_json::from_str(&json).map_err(|e| { - StorageError::ReadFailed(format!("Failed to deserialize peers reputations: {e}")) - }) - } } #[cfg(test)] diff --git a/dash-spv/src/storage/segments.rs b/dash-spv/src/storage/segments.rs index e54286c93..17e4f8dd1 100644 --- a/dash-spv/src/storage/segments.rs +++ b/dash-spv/src/storage/segments.rs @@ -178,14 +178,6 @@ impl SegmentCache { } } - if let Some(segment_id) = max_seg_id { - let segment = cache.get_segment(&segment_id).await?; - - cache.tip_height = segment - .last_valid_offset() - .map(|offset| Self::segment_id_to_start_height(segment_id) + offset); - } - if let Some(segment_id) = min_seg_id { let segment = cache.get_segment(&segment_id).await?; @@ -193,6 +185,57 @@ impl SegmentCache { .first_valid_offset() .map(|offset| Self::segment_id_to_start_height(segment_id) + offset); } + + // Reopen at the DURABLE CONTIGUOUS tip — the highest height with no hole + // below it — not the highest stored height. Out-of-order stores can leave + // a hole a crash then freezes in place (8000-8999 present, 7000-7999 not); + // trusting `last_valid_offset` there would resume above the hole and never + // refill it, silently dropping the wallet's view of that range. Walk the + // segments in order and stop at the first gap. + if let (Some(min_id), Some(max_id)) = (min_seg_id, max_seg_id) { + let raw_tip = { + let segment = cache.get_segment(&max_id).await?; + segment + .last_valid_offset() + .map(|offset| Self::segment_id_to_start_height(max_id) + offset) + }; + cache.tip_height = raw_tip; + + let mut contiguous_tip = None; + let mut next_expected: Option = None; + for seg_id in min_id..=max_id { + let base = Self::segment_id_to_start_height(seg_id); + let segment = cache.get_segment(&seg_id).await?; + let Some(first) = segment.first_valid_offset() else { + break; // an empty segment is itself a gap + }; + // A gap between the previous segment's end and this one's start. + if next_expected.is_some_and(|exp| base + first != exp) { + break; + } + let last = segment.contiguous_last_offset().expect("has a valid item"); + contiguous_tip = Some(base + last); + // A hole inside this segment ends the contiguous prefix here. + if last + 1 < Segment::::ITEMS_PER_SEGMENT { + break; + } + next_expected = Some(base + last + 1); + } + + // Physically drop everything above the contiguous tip. Leaving the + // orphaned data in place would resurface two ways: the re-download + // stores back over occupied slots — which `Segment::insert` forbids + // (debug_assert) — and a later reopen would recompute a raw tip that + // skips the hole again. Truncating clears the slots to sentinels so + // the refill lands cleanly and the on-disk state matches the tip. + if let Some(ct) = contiguous_tip { + if raw_tip.is_some_and(|rt| rt > ct) { + cache.truncate_above(ct).await?; + } + } else { + cache.tip_height = None; + } + } } Ok(cache) @@ -658,6 +701,29 @@ impl Segment { None } + /// Offset of the last item in the CONTIGUOUS run that starts at + /// [`first_valid_offset`] — i.e. the first hole stops the run. + /// + /// Filters (and blocks) are stored as they arrive, which is out of order, so a + /// segment can hold a hole: 8000-8999 present while 7000-7999 is not. Reopening + /// at [`last_valid_offset`] (8999) would silently skip that hole — resume would + /// download from 9000 and the wallet would never see the missing range. This is + /// the tip that never claims past a hole, so resume refills it in order. + /// + /// `None` if the segment is empty. + pub fn contiguous_last_offset(&self) -> Option { + let sentinel = I::sentinel(); + let first = self.first_valid_offset()? as usize; + let mut last = first; + for (index, item) in self.items.iter().enumerate().skip(first) { + if item == &sentinel { + break; + } + last = index; + } + Some(last as u32) + } + pub async fn load(base_path: &Path, segment_id: u32) -> StorageResult { // Load segment from disk let segment_path = base_path.join(I::segment_file_name(segment_id)); diff --git a/dash-spv/src/sync/block_headers/manager.rs b/dash-spv/src/sync/block_headers/manager.rs index 9844a93cb..80c765595 100644 --- a/dash-spv/src/sync/block_headers/manager.rs +++ b/dash-spv/src/sync/block_headers/manager.rs @@ -13,7 +13,7 @@ use std::time::Instant; use crate::chain::CheckpointManager; use crate::error::{SyncError, SyncResult}; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, BlockHeaderTip, MetadataStorage}; use crate::sync::block_headers::HeadersPipeline; use crate::sync::{BlockHeadersProgress, ProgressPercentage, SyncEvent, SyncManager, SyncState}; diff --git a/dash-spv/src/sync/block_headers/pipeline.rs b/dash-spv/src/sync/block_headers/pipeline.rs index 4cf034ac3..67ea51968 100644 --- a/dash-spv/src/sync/block_headers/pipeline.rs +++ b/dash-spv/src/sync/block_headers/pipeline.rs @@ -6,13 +6,11 @@ use std::sync::Arc; -#[cfg(test)] -use dashcore::block::Header; use dashcore::BlockHash; use crate::chain::CheckpointManager; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::sync::block_headers::segment_state::SegmentState; use crate::types::HashedBlockHeader; @@ -151,18 +149,8 @@ impl HeadersPipeline { Ok(sent) } - /// Try to match incoming headers to the correct segment. - /// Returns the segment index if matched, or None if headers don't belong to any segment. - /// Returns an error if checkpoint validation fails. - #[cfg(test)] - pub fn receive_headers(&mut self, headers: &[Header]) -> SyncResult> { - let hashed: Vec = headers.iter().map(HashedBlockHeader::from).collect(); - Ok(self.receive_headers_prehashed(&hashed)?.map(|(sid, ..)| sid)) - } - /// Match an already-hashed batch to the correct segment and route it. The X11 - /// block hashing is done in parallel at the manager (on the tokio runtime); - /// this is the production path, `receive_headers` is a test-only wrapper. + /// block hashing is done in parallel at the manager (on the tokio runtime). pub fn receive_headers_prehashed( &mut self, hashed: &[HashedBlockHeader], @@ -349,12 +337,4 @@ impl HeadersPipeline { } false } - - /// Check if the tip segment has active requests in flight. - pub fn tip_segment_has_pending_request(&self) -> bool { - self.segments - .iter() - .find(|s| s.target_height.is_none()) - .is_some_and(|s| !s.complete && s.coordinator.active_count() > 0) - } } diff --git a/dash-spv/src/sync/block_headers/segment_state.rs b/dash-spv/src/sync/block_headers/segment_state.rs index 094f562ec..7a6cbfbfc 100644 --- a/dash-spv/src/sync/block_headers/segment_state.rs +++ b/dash-spv/src/sync/block_headers/segment_state.rs @@ -1,13 +1,11 @@ use crate::error::{SyncError, SyncResult}; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::sync::download_coordinator::DownloadCoordinator; use crate::types::HashedBlockHeader; use dashcore::hashes::Hash; use dashcore::network::message::NetworkMessage; use dashcore::network::message_blockdata::GetHeadersMessage; use dashcore::BlockHash; -#[cfg(test)] -use dashcore::Header; use std::sync::Arc; /// State for a single download segment between two checkpoints. @@ -104,17 +102,6 @@ impl SegmentState { &self.current_tip_hash == prev_blockhash } - /// Process received headers for this segment. - /// Returns the number of headers processed, or an error if checkpoint validation fails. - #[cfg(test)] - pub(super) fn receive_headers(&mut self, headers: &[Header]) -> SyncResult { - let hashed: Vec = headers.iter().map(HashedBlockHeader::from).collect(); - Ok(self - .receive_headers_prehashed(&hashed)? - .map(|(s, e, _)| (e - s + 1) as usize) - .unwrap_or(0)) - } - /// Route an already-hashed batch to this segment. The expensive X11 block /// hashing is done in parallel off this task (at the manager, on the tokio /// runtime), so here we only run the cheap sequential chain/checkpoint check. diff --git a/dash-spv/src/sync/block_headers/sync_manager.rs b/dash-spv/src/sync/block_headers/sync_manager.rs index ba2034499..e37b7e7f7 100644 --- a/dash-spv/src/sync/block_headers/sync_manager.rs +++ b/dash-spv/src/sync/block_headers/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, MetadataStorage}; use crate::sync::sync_manager::ensure_not_started; use crate::sync::{ @@ -39,13 +39,13 @@ impl SyncManager for BlockHeadersMana self.progress.update_target_height(height); } - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { - use crate::network2::MessageType; + fn subscribed_commands(&self) -> &'static [crate::network::MessageType] { + use crate::network::MessageType; &[MessageType::Headers, MessageType::Inv] } - fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { - if let crate::network2::RequestKey::Headers(hash) = key { + fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { + if let crate::network::RequestKey::Headers(hash) = key { self.pipeline.mark_on_flight(hash); } } diff --git a/dash-spv/src/sync/blocks/manager.rs b/dash-spv/src/sync/blocks/manager.rs index 47538f06d..5fef6cafd 100644 --- a/dash-spv/src/sync/blocks/manager.rs +++ b/dash-spv/src/sync/blocks/manager.rs @@ -9,8 +9,8 @@ use tokio::sync::RwLock; use super::pipeline::BlocksPipeline; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; -use crate::storage::{BlockHeaderStorage, BlockStorage}; +use crate::network::PeerNetworkManager; +use crate::storage::BlockStorage; use crate::sync::{BlocksProgress, SyncEvent, SyncManager, SyncState}; use key_wallet_manager::WalletInterface; @@ -23,14 +23,11 @@ use key_wallet_manager::WalletInterface; /// - Emits BlockProcessed events /// /// Generic over: -/// - `H: BlockHeaderStorage` for height lookups /// - `B: BlockStorage` for storing and loading blocks /// - `W: WalletInterface` for wallet operations -pub struct BlocksManager { +pub struct BlocksManager { /// Current progress of the manager. pub(super) progress: BlocksProgress, - /// Block header storage (for height lookups). - pub(super) header_storage: Arc>, /// Block storage (for storing and loading blocks). pub(super) block_storage: Arc>, /// Wallet for processing blocks. @@ -41,13 +38,9 @@ pub struct BlocksManager BlocksManager { +impl BlocksManager { /// Create a new blocks manager with the given storage references. - pub async fn new( - wallet: Arc>, - header_storage: Arc>, - block_storage: Arc>, - ) -> Self { + pub async fn new(wallet: Arc>, block_storage: Arc>) -> Self { let last_processed_height = wallet.read().await.last_processed_height(); let mut initial_progress = BlocksProgress::default(); @@ -55,7 +48,6 @@ impl BlocksManager BlocksManager std::fmt::Debug - for BlocksManager -{ +impl std::fmt::Debug for BlocksManager { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("BlocksManager") .field("progress", &self.progress) diff --git a/dash-spv/src/sync/blocks/pipeline.rs b/dash-spv/src/sync/blocks/pipeline.rs index 2c684d73b..9f154fa73 100644 --- a/dash-spv/src/sync/blocks/pipeline.rs +++ b/dash-spv/src/sync/blocks/pipeline.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::sync::download_coordinator::DownloadCoordinator; use crate::types::HashedBlock; use dashcore::network::message::NetworkMessage; @@ -143,21 +143,6 @@ impl BlocksPipeline { Ok(hashes.len()) } - /// Bytes of blocks downloaded and waiting in RAM for their turn to be processed - /// (the wallet must see them in height order). Up to ~2MB each — the reason this - /// buffer is worth watching. - pub(super) fn buffered_bytes(&self) -> usize { - self.downloaded - .values() - .map(|b| dashcore::consensus::encode::serialize(b.block()).len()) - .sum() - } - - /// Number of blocks downloaded and waiting to be processed. - pub(super) fn buffered_blocks(&self) -> usize { - self.downloaded.len() - } - /// Handle a received block using internal height mapping. /// /// Stores the block in the downloaded buffer for height-ordered processing and diff --git a/dash-spv/src/sync/blocks/sync_manager.rs b/dash-spv/src/sync/blocks/sync_manager.rs index 0f7be55bc..8030328b9 100644 --- a/dash-spv/src/sync/blocks/sync_manager.rs +++ b/dash-spv/src/sync/blocks/sync_manager.rs @@ -1,6 +1,6 @@ use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; -use crate::storage::{BlockHeaderStorage, BlockStorage}; +use crate::network::PeerNetworkManager; +use crate::storage::BlockStorage; use crate::sync::sync_manager::ensure_not_started; use crate::sync::{ BlocksManager, ManagerIdentifier, SyncEvent, SyncManager, SyncManagerProgress, SyncState, @@ -15,9 +15,7 @@ use std::net::SocketAddr; use std::sync::Arc; #[async_trait] -impl SyncManager - for BlocksManager -{ +impl SyncManager for BlocksManager { fn identifier(&self) -> ManagerIdentifier { ManagerIdentifier::Block } @@ -30,14 +28,14 @@ impl SyncM self.progress.set_state(state); } - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { - &[crate::network2::MessageType::Block] + fn subscribed_commands(&self) -> &'static [crate::network::MessageType] { + &[crate::network::MessageType::Block] } /// The router put a block request on the wire: start ITS response timeout from /// here, not from when it was queued — the queue can hold it for a while. - fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { - if let crate::network2::RequestKey::Block(hash) = key { + fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { + if let crate::network::RequestKey::Block(hash) = key { self.pipeline.mark_on_flight(hash); } } diff --git a/dash-spv/src/sync/chainlock/sync_manager.rs b/dash-spv/src/sync/chainlock/sync_manager.rs index c9bd94847..c54b6a6fb 100644 --- a/dash-spv/src/sync/chainlock/sync_manager.rs +++ b/dash-spv/src/sync/chainlock/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, MetadataStorage}; use crate::sync::{ ChainLockManager, ManagerIdentifier, SyncEvent, SyncManager, SyncManagerProgress, SyncState, @@ -24,8 +24,8 @@ impl SyncManager for ChainLockManager self.progress.set_state(state); } - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { - use crate::network2::MessageType; + fn subscribed_commands(&self) -> &'static [crate::network::MessageType] { + use crate::network::MessageType; &[MessageType::ChainLock, MessageType::Inv] } @@ -36,7 +36,7 @@ impl SyncManager for ChainLockManager async fn handle_message( &mut self, - peer: SocketAddr, + _peer: SocketAddr, msg: NetworkMessage, network: &Arc, ) -> SyncResult> { diff --git a/dash-spv/src/sync/filter_headers/manager.rs b/dash-spv/src/sync/filter_headers/manager.rs index fe6933075..6bad21dec 100644 --- a/dash-spv/src/sync/filter_headers/manager.rs +++ b/dash-spv/src/sync/filter_headers/manager.rs @@ -11,7 +11,7 @@ use tokio::sync::RwLock; use super::pipeline::FilterHeadersPipeline; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage}; use crate::sync::filter_headers::util::compute_filter_headers; use crate::sync::progress::ProgressPercentage; diff --git a/dash-spv/src/sync/filter_headers/pipeline.rs b/dash-spv/src/sync/filter_headers/pipeline.rs index 37c6a527e..4d36780e2 100644 --- a/dash-spv/src/sync/filter_headers/pipeline.rs +++ b/dash-spv/src/sync/filter_headers/pipeline.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; use std::sync::Arc; use crate::error::{SyncError, SyncResult}; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::BlockHeaderStorage; use crate::sync::download_coordinator::DownloadCoordinator; diff --git a/dash-spv/src/sync/filter_headers/sync_manager.rs b/dash-spv/src/sync/filter_headers/sync_manager.rs index 8366bd7c8..9cf2b519e 100644 --- a/dash-spv/src/sync/filter_headers/sync_manager.rs +++ b/dash-spv/src/sync/filter_headers/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage}; use crate::sync::filter_headers::pipeline::FilterHeadersPipeline; use crate::sync::progress::ProgressPercentage; @@ -30,12 +30,12 @@ impl SyncManager for FilterHeade self.progress.update_target_height(height); } - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { - &[crate::network2::MessageType::CfHeaders] + fn subscribed_commands(&self) -> &'static [crate::network::MessageType] { + &[crate::network::MessageType::CfHeaders] } - fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { - if let crate::network2::RequestKey::CfHeaders(stop_hash) = key { + fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { + if let crate::network::RequestKey::CfHeaders(stop_hash) = key { self.pipeline.mark_on_flight(stop_hash); } } @@ -48,7 +48,7 @@ impl SyncManager for FilterHeade async fn handle_message( &mut self, - peer: SocketAddr, + _peer: SocketAddr, msg: NetworkMessage, network: &Arc, ) -> SyncResult> { diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index cdfb12538..6529aef4f 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -14,14 +14,19 @@ use super::batch::FiltersBatch; use super::block_match_tracker::{BlockMatchTracker, BlockTrackResult}; use super::pipeline::FiltersPipeline; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; -use crate::storage::{BlockHeaderStorage, FilterHeaderStorage, FilterStorage}; +use crate::network::PeerNetworkManager; +use crate::storage::{ + BlockHeaderStorage, FilterHeaderStorage, FilterStorage, MetadataStorage, + PersistentMetadataStorage, +}; use crate::sync::filters::util::get_prev_filter_header; use crate::sync::{FiltersProgress, SyncEvent, SyncManager, SyncState}; use crate::validation::{FilterValidationInput, FilterValidator, Validator}; use crate::sync::progress::ProgressPercentage; use dashcore::hash_types::FilterHeader; +use dashcore::hashes::Hash; +use dashcore::BlockHash; use key_wallet_manager::WalletInterface; use key_wallet_manager::{FilterMatchKey, WalletId}; use tokio::sync::RwLock; @@ -29,6 +34,226 @@ use tokio::sync::RwLock; /// Batch size for processing filters. const BATCH_PROCESSING_SIZE: u32 = 5000; +/// Metadata key holding the durable scan state (see [`ScanState`]). +const SCAN_STATE_KEY: &str = "filters_scan_state"; + +/// How far the filter scan has got, and which matched blocks it is still owed. +/// +/// The wallet's `synced_height` only advances once a batch's matched blocks have +/// all been PROCESSED, so on its own it cannot say "these filters are already +/// scanned, I am merely waiting for blocks". Without that distinction a restart +/// rewinds the scan to `synced_height` and re-matches every stored filter from +/// scratch — which re-emits `BlocksNeeded` for blocks already requested, in a +/// burst, before a single block round-trip can complete. A client restarted +/// faster than that round-trip then makes NO durable progress, ever. +/// +/// So we record the scan frontier separately, plus the matched blocks it is +/// waiting on. On restart the scan resumes at `watermark + 1` and the pending +/// blocks are re-requested (served straight from block storage if they were +/// already downloaded). +/// +/// The frontier is tracked PER WALLET, because scanning is: a batch records the +/// `scanned_wallets` it was matched for, and only those wallets get their +/// `synced_height` advanced when it commits. A single global frontier would let +/// a wallet added later inherit another wallet's progress and skip the filters +/// it was never actually matched against — silently losing its transactions. A +/// wallet absent from the map simply has no frontier, so it rescans from its own +/// `synced_height`, which is exactly what a fresh wallet needs. +/// +/// This is a CACHE, never a source of truth: it is only ever consulted to skip +/// work a wallet has already been credited for. Losing it (a hard kill, a corrupt +/// file) costs a rescan and nothing else — it can never make the wallet skip a +/// block, because `synced_height` itself still only advances through the normal +/// commit path, once the matched blocks are processed. +#[derive(Debug, Clone, Copy)] +pub(super) struct WalletFrontier { + /// Every filter at or below this height has been scanned for this wallet, and + /// any match is recorded in `ScanState::pending` (or has already been processed). + scanned: u32, + /// The wallet's `synced_height` when `scanned` was last moved. `synced_height` + /// only ever moves forward, so finding it BELOW this value means the wallet was + /// rolled back or rebuilt underneath us — and a frontier describing a scan of a + /// wallet that no longer exists in that form must not be trusted. Note a wallet + /// legitimately sits below its own frontier all the time (that is precisely the + /// gap this type exists to describe: scanned, blocks still in flight), so the + /// frontier alone cannot tell the two apart. This can. + synced_at: u32, + /// The count of monitored scripts+elements this wallet had when `scanned` was + /// last moved. Gap-limit derivation only ever GROWS this set, so finding the + /// wallet with MORE monitored scripts than this means addresses were derived + /// since the range was scanned — and a filter already below the frontier may + /// match one of them (a tx to an address not yet derived when its block was + /// first scanned). `note_new_scripts` rolls the frontier back in-run when it + /// sees this, but that reaction rides a transient `BlockProcessed` event that a + /// restart can drop while the derived address is already durable in the wallet. + /// Recording the count lets `rescan_grown_scripts` reconstruct the rollback + /// from durable state on resume — the wallet's script count is recomputed from + /// its persisted address pools, so it is comparable across restarts even though + /// the in-memory `monitor_revision` counter is not. + scanned_scripts: u32, +} + +#[derive(Debug, Default, Clone)] +pub(super) struct ScanState { + /// Per-wallet scan frontier. + watermarks: BTreeMap, + /// Matched blocks awaiting `BlockProcessed`, keyed by height. + pending: BTreeMap)>, +} + +impl ScanState { + /// The highest height already scanned for EVERY one of `wallets`, if there is + /// one. `None` as soon as a single wallet has no recorded frontier — a wallet + /// added since the last run has been matched against nothing, so the scan may + /// not skip a single filter on its behalf, whatever the others have covered. + fn covered_through(&self, wallets: &BTreeSet) -> Option { + if wallets.is_empty() { + return None; + } + wallets.iter().map(|id| self.watermarks.get(id).map(|f| f.scanned)).min().flatten() + } + + /// Roll a wallet's scan frontier back to `height`, so the range above it is + /// rescanned. Processing a matched block can derive fresh gap-limit scripts, + /// and a filter ALREADY scanned (below the frontier) may match one of them — + /// a transaction to an address that had not been derived when its block was + /// first scanned. The frontier otherwise skips that range forever, silently + /// dropping the transaction; lowering it forces the rescan that finds it. + fn rollback_wallet(&mut self, wallet_id: &WalletId, height: u32) { + if let Some(f) = self.watermarks.get_mut(wallet_id) { + if f.scanned > height { + f.scanned = height; + } + } + } + + /// Drop the frontier of any wallet that has gone backwards (see + /// [`WalletFrontier::synced_at`]), along with the matched blocks it alone was + /// waiting on — the rescan those wallets now need will re-derive them. + fn drop_rolled_back(&mut self, synced_now: &BTreeMap) { + let stale: BTreeSet = self + .watermarks + .iter() + .filter(|(id, f)| synced_now.get(*id).copied().unwrap_or(0) < f.synced_at) + .map(|(id, _)| *id) + .collect(); + if stale.is_empty() { + return; + } + tracing::info!("Discarding filter scan frontier for {} rolled-back wallet(s)", stale.len()); + for id in &stale { + self.watermarks.remove(id); + } + self.pending.retain(|_, (_, wallets)| { + wallets.retain(|w| !stale.contains(w)); + !wallets.is_empty() + }); + } + + /// Roll a wallet's frontier back to `birth` when it now monitors MORE scripts + /// than when its `scanned` range was covered. Reconstructs, from durable wallet + /// state, the `note_new_scripts` rollback that a restart can lose: the derived + /// address is already persisted in the wallet, but the `BlockProcessed` event + /// that would have rolled the frontier back is transient. Without this, a filter + /// below the frontier that matches a freshly-derived address is skipped forever + /// — the wallet silently loses that transaction. Returns the wallets rolled back. + /// + /// `script_counts` is each present wallet's current monitored-script count. A + /// wallet absent from it is treated as 0 (no scripts, nothing to rescan). The + /// count only grows through gap-limit derivation, so `current > scanned_scripts` + /// is exactly "addresses derived since the scan". A rolled-back wallet (fewer + /// scripts) is handled separately by `drop_rolled_back`. + fn rescan_grown_scripts( + &mut self, + script_counts: &BTreeMap, + birth: u32, + ) -> BTreeSet { + let mut rolled = BTreeSet::new(); + for (id, f) in self.watermarks.iter_mut() { + let current = script_counts.get(id).copied().unwrap_or(0); + if current > f.scanned_scripts && f.scanned > birth { + f.scanned = birth; + rolled.insert(*id); + } + } + rolled + } + + /// `n_watermarks | [wallet_id, height] | n_pending | [height, hash, n_wallets, wallet_ids...]`, + /// all little-endian. + fn encode(&self) -> Vec { + let mut out = Vec::with_capacity(8 + self.watermarks.len() * 40 + self.pending.len() * 48); + out.extend_from_slice(&(self.watermarks.len() as u32).to_le_bytes()); + for (id, f) in &self.watermarks { + out.extend_from_slice(id); + out.extend_from_slice(&f.scanned.to_le_bytes()); + out.extend_from_slice(&f.synced_at.to_le_bytes()); + out.extend_from_slice(&f.scanned_scripts.to_le_bytes()); + } + out.extend_from_slice(&(self.pending.len() as u32).to_le_bytes()); + for (height, (hash, wallets)) in &self.pending { + out.extend_from_slice(&height.to_le_bytes()); + out.extend_from_slice(hash.as_byte_array()); + out.extend_from_slice(&(wallets.len() as u32).to_le_bytes()); + for w in wallets { + out.extend_from_slice(w); + } + } + out + } + + /// Returns `None` on any malformed input: a corrupt cache must degrade to a + /// rescan, never to a half-decoded pending set that could silently drop a + /// block the wallet still needs. + fn decode(bytes: &[u8]) -> Option { + let mut r = bytes; + let mut take = |n: usize| -> Option<&[u8]> { + if r.len() < n { + return None; + } + let (head, tail) = r.split_at(n); + r = tail; + Some(head) + }; + + let n_watermarks = u32::from_le_bytes(take(4)?.try_into().ok()?); + let mut watermarks = BTreeMap::new(); + for _ in 0..n_watermarks { + let id: WalletId = take(32)?.try_into().ok()?; + let scanned = u32::from_le_bytes(take(4)?.try_into().ok()?); + let synced_at = u32::from_le_bytes(take(4)?.try_into().ok()?); + let scanned_scripts = u32::from_le_bytes(take(4)?.try_into().ok()?); + watermarks.insert( + id, + WalletFrontier { + scanned, + synced_at, + scanned_scripts, + }, + ); + } + + let n_pending = u32::from_le_bytes(take(4)?.try_into().ok()?); + let mut pending = BTreeMap::new(); + for _ in 0..n_pending { + let height = u32::from_le_bytes(take(4)?.try_into().ok()?); + let hash = BlockHash::from_slice(take(32)?).ok()?; + let n_wallets = u32::from_le_bytes(take(4)?.try_into().ok()?); + let mut wallets = BTreeSet::new(); + for _ in 0..n_wallets { + let id: WalletId = take(32)?.try_into().ok()?; + wallets.insert(id); + } + pending.insert(height, (hash, wallets)); + } + + Some(Self { + watermarks, + pending, + }) + } +} + /// Maximum shards a batch's filters are split into for parallel matching. const MATCH_SHARD_CAP: usize = 16; @@ -237,6 +462,17 @@ pub struct FiltersManager< filter_header_storage: Arc>, /// Filter storage (for storing filters). pub(super) filter_storage: Arc>, + /// Metadata storage, holding the durable [`ScanState`]. + metadata_storage: Arc>, + /// Durable scan frontier and the matched blocks it is still owed. Lets a + /// restart resume the scan instead of re-matching from `synced_height`. + pub(super) scan: ScanState, + /// `scan` has changed since it was last written. Persisting is a whole-map + /// atomic write, far too costly to do on every matched/processed block on + /// mainnet, so mutators only flip this flag; the write is coalesced into the + /// periodic tick and forced on shutdown. Safe because `scan` is a cache — + /// bounded staleness costs a bounded rescan, never a skipped block. + scan_dirty: bool, /// Wallet for matching filters. pub(super) wallet: Arc>, /// Pipeline for downloading filters. @@ -282,6 +518,13 @@ pub struct FiltersManager< /// reach, so they arrive around the same time as these do — keeping them here /// means the verification never has to read back what we were just handed. pub(super) filter_headers: BTreeMap, + /// Last time the throttled MEM debug line was emitted (once per 2s). + last_mem_log: Option, + /// Cached wallet birth (`earliest_required_height`), refreshed each + /// `start_download`. Lets the BlockProcessed handler roll a wallet's frontier + /// back to birth WITHOUT taking a wallet lock — doing that read inside the + /// handler deadlocks against block processing under rapid restart. + pub(super) birth_height: u32, } impl @@ -293,8 +536,16 @@ impl>, filter_header_storage: Arc>, filter_storage: Arc>, + metadata_storage: Arc>, ) -> Self { let committed_height = wallet.read().await.synced_height(); + let scan = match metadata_storage.read().await.load_metadata(SCAN_STATE_KEY).await { + Ok(Some(bytes)) => ScanState::decode(&bytes).unwrap_or_else(|| { + tracing::warn!("Corrupt filter scan state; rescanning from synced height"); + ScanState::default() + }), + _ => ScanState::default(), + }; let stored_height = filter_storage.read().await.filter_tip_height().await.unwrap_or(0); let target_height = header_storage.read().await.get_tip().await.map(|t| t.height()).unwrap_or(0); @@ -318,6 +569,9 @@ impl self.scan_dirty = false, + // Non-fatal: the frontier is a cache, and losing it only costs a rescan. + // Leave `scan_dirty` set so the next tick retries. + Err(e) => tracing::warn!("Failed to persist filter scan state: {e}"), + } + } + + /// Record a matched block the scan is now waiting on. + fn note_pending_block(&mut self, key: &FilterMatchKey, wallets: &BTreeSet) { + self.scan + .pending + .entry(key.height()) + .or_insert_with(|| (*key.hash(), BTreeSet::new())) + .1 + .extend(wallets.iter().copied()); + self.scan_dirty = true; + } + + /// Advance each scanned wallet's frontier to `end`, now that every match in + /// `start..=end` is recorded in `pending`. Called when a batch finishes + /// scanning — NOT when it commits — which is the whole point: the frontier + /// has to survive a restart that happens while the blocks are still in flight. + /// `synced_now` is each wallet's `synced_height` as of this scan; it is stored + /// with the frontier so a later run can tell a wallet that is merely awaiting + /// blocks from one that has been rolled back. See [`WalletFrontier::synced_at`]. + /// `script_counts` is each wallet's monitored-script count for this SAME scan + /// snapshot; recording it lets a later run detect addresses derived since (see + /// [`WalletFrontier::scanned_scripts`] and [`ScanState::rescan_grown_scripts`]). + fn advance_frontier( + &mut self, + end: u32, + wallets: &BTreeSet, + synced_now: &BTreeMap, + script_counts: &BTreeMap, + ) { + for id in wallets { + let synced = synced_now.get(id).copied().unwrap_or(0); + let scripts = script_counts.get(id).copied().unwrap_or(0); + let f = self.scan.watermarks.entry(*id).or_insert(WalletFrontier { + scanned: 0, + synced_at: synced, + scanned_scripts: scripts, + }); + f.scanned = f.scanned.max(end); + f.synced_at = f.synced_at.max(synced); + // This pass scanned `end` against exactly `scripts` monitored scripts, + // so record that — NOT a max. A later count above this is what flags a + // range scanned before newer addresses were derived. + f.scanned_scripts = scripts; + } + self.scan_dirty = true; + } + + /// Drop a matched block from the pending set once it has been processed. + pub(super) fn clear_pending_block(&mut self, height: u32) { + if self.scan.pending.remove(&height).is_some() { + self.scan_dirty = true; + } + } + + /// A processed block derived fresh gap-limit scripts for `wallet_id`. Any filter + /// already scanned for it may match one of those scripts (a transaction to an + /// address not yet derived when that filter was first scanned), so roll the + /// wallet's frontier back to its birth: the next scan pass re-matches the whole + /// stored range against the grown script set and requests any block it turns up. + /// The reload is from disk — the filters are already stored — so this re-examines + /// without re-downloading. + pub(super) fn note_new_scripts(&mut self, wallet_id: &WalletId, birth_height: u32) { + self.scan.rollback_wallet(wallet_id, birth_height); + self.scan_dirty = true; + } + + /// Take ownership of `committed_start..scan_start`: a range an earlier run + /// scanned but never COMMITTED, because the commit is what moves the wallets' + /// `synced_height` and that run died before its matched blocks landed. + /// + /// The resumed scan starts ABOVE this range (that is the whole point of the + /// frontier), so nothing else will ever look at it again. Without an owner the + /// wallets could never be credited for it and the sync would stall short of the + /// tip forever. A stand-in batch spanning it does the job: it re-requests the + /// blocks that run left in flight — served straight from block storage when they + /// were already downloaded — and commits, crediting every wallet the frontier + /// covers, once the last one is processed. With nothing outstanding it commits + /// immediately, which is equally correct: the frontier says those filters were + /// scanned, and an empty `pending` says nothing in them matched. + fn adopt_uncommitted_range( + &mut self, + committed_start: u32, + scan_start: u32, + wallets: &BTreeSet, + ) -> Vec { + if committed_start >= scan_start { + return Vec::new(); + } + let (start, end) = (committed_start, scan_start - 1); + + let mut blocks: BTreeMap> = BTreeMap::new(); + let mut in_flight = 0u32; + for (height, (hash, owners)) in self.scan.pending.clone() { + if height > end { + continue; + } + let key = FilterMatchKey::new(height, hash); + match self.tracker.track(&key, start, owners) { + BlockTrackResult::NewlyTracked { + wallets, + } => { + blocks.insert(key, wallets); + in_flight += 1; + } + BlockTrackResult::InFlight { + wallets, + } => { + blocks.insert(key, wallets); + } + BlockTrackResult::AlreadyProcessed => {} + } + } + + // Credit every wallet the frontier covers, not just those that happen to own + // a pending block: `scan_start - 1` is the LOWEST frontier across them all, + // so the range was scanned for each of them. + let mut recovery = FiltersBatch::new(start, end, HashMap::new()); + recovery.mark_verified(); + recovery.mark_scanned(); + recovery.set_pending_blocks(in_flight); + recovery.set_scanned_wallets(wallets.clone()); + self.active_batches.insert(start, recovery); + + tracing::info!( + "Resuming scanned-but-uncommitted range {}-{} ({} matched block(s) still owed)", + start, + end, + in_flight + ); + + if blocks.is_empty() { + Vec::new() + } else { + vec![SyncEvent::BlocksNeeded { + blocks, + }] + } } async fn load_filters( @@ -396,10 +812,24 @@ impl = + ids.iter().map(|id| (*id, wallet.wallet_synced_height(id))).collect(); + // Current monitored-script count per wallet, to detect addresses derived + // since the frontier last scanned each range (see `rescan_grown_scripts`). + let script_counts: BTreeMap = ids + .iter() + .map(|id| { + let n = wallet.monitored_script_pubkeys_for(id).len() + + wallet.monitored_filter_elements_for(id).len(); + (*id, n as u32) + }) + .collect(); + (wallet.earliest_required_height().await, wallet.synced_height(), synced, script_counts) }; + self.birth_height = wallet_birth_height; // Get stored filters tip let stored_filters_tip = self.filter_storage.read().await.filter_tip_height().await?; @@ -410,13 +840,66 @@ impl 0 { + let committed_start = if wallet_committed_height > 0 { wallet_birth_height.max(wallet_committed_height + 1) } else { wallet_birth_height } .max(header_start_height); + // Resume from the durable scan frontier when it reaches beyond what the + // wallets have been credited for: those filters have already been matched + // and whatever they hit is in `scan.pending`, re-requested below. Without + // this the scan rewinds to `synced_height` on every start and re-matches + // the whole stored range in one burst — and a client stopped before its + // matched blocks land (they need a network round-trip; the rescan needs + // none) never makes any durable progress at all. + // + // A wallet that has gone backwards (rebuilt, rolled back) invalidates the + // frontier recorded for it: it describes a scan of a wallet that no longer + // exists in that form, and trusting it would skip filters that wallet has + // never actually been matched against. + self.scan.drop_rolled_back(&wallet_synced); + + // A restart can drop the transient `BlockProcessed` event that would have + // rolled the frontier back for gap-limit scripts a processed block derived + // (`note_new_scripts`), while the derived address is already durable in the + // wallet. Reconstruct that rollback from durable state: any wallet now + // monitoring more scripts than its frontier was scanned against is rolled + // back to birth, so the range below the frontier is rescanned against the + // grown set and a transaction to a freshly-derived address is not skipped + // for good. In steady state (no new scripts) this is a no-op, so the + // frontier's fast resume — and its protection against rescan starvation — + // is untouched. + let rescanned = self.scan.rescan_grown_scripts(&script_counts, wallet_birth_height); + if !rescanned.is_empty() { + self.scan_dirty = true; + tracing::info!( + "Filter scan: {} wallet(s) derived scripts since their last scan; \ + rescanning from birth {} to catch transactions to the new addresses", + rescanned.len(), + wallet_birth_height, + ); + } + + // `covered_through` yields the LOWEST frontier across the current wallets, + // and nothing at all if any of them has none — a wallet added since the + // last run has been matched against no filter, so the scan must not skip + // any on its behalf, however far the others got. + // + // Cap it at the durable filter tip: the frontier is persisted independently + // of the filters, so a crash can leave it claiming a range whose filters the + // gap-aware reopen has since rolled back (a hole below what the frontier + // recorded). Scanning may only skip what is BOTH matched and still on disk; + // beyond `stored_filters_tip` the filters must be re-downloaded and rescanned. + let ids: BTreeSet = wallet_synced.keys().copied().collect(); + let scan_start = match self.scan.covered_through(&ids) { + Some(covered) => { + committed_start.max(covered.min(stored_filters_tip) + 1).max(header_start_height) + } + None => committed_start, + }; + // Determine download start (where we need to download from) // Must be at least header_start_height for checkpoint-based sync let download_start = if stored_filters_tip > 0 { @@ -434,26 +917,50 @@ impl self.progress.filter_header_tip_height() { // Park the idle pipeline at the download frontier so a later // `extend_target` from a new block queues from here instead of // re-requesting every filter from an uninitialized position. self.filter_pipeline.init(download_start, download_start.saturating_sub(1)); - // Only emit FiltersSyncComplete if we've also reached the chain tip - // This prevents premature sync complete while filter headers are still syncing - if self.progress.committed_height() >= self.progress.target_height() { + + // Settle the adopted range now: with its blocks already processed (or + // none matched at all) it commits straight away, which is what carries + // the wallets the last stretch to the tip. + events.extend(self.try_commit_batches().await?); + + // Only emit FiltersSyncComplete if we've also reached the chain tip AND + // no matched block is still awaiting processing — otherwise a resume that + // just re-adopted pending blocks would report done while the wallet is + // still missing their transactions. + if self.progress.committed_height() >= self.progress.target_height() + && self.scan.pending.is_empty() + { self.set_state(SyncState::Synced); tracing::info!("Filters already synced to {}", self.progress.target_height()); - return Ok(vec![SyncEvent::FiltersSyncComplete { + events.push(SyncEvent::FiltersSyncComplete { tip_height: self.progress.committed_height(), - }]); + }); + return Ok(events); } // Not enough filter headers yet to start scanning. Go back to waiting // so the next FilterHeadersStored event triggers start_download again // with proper batch processing initialization. self.set_state(SyncState::WaitForEvents); - return Ok(vec![]); + return Ok(events); } tracing::info!( @@ -504,11 +1011,11 @@ impl= batch_end { - self.scan_batch(scan_start).await + events.extend(self.scan_batch(scan_start).await?); + Ok(events) } else { tracing::debug!( "Initial batch {}-{}: waiting for filters (stored_height={})", @@ -516,7 +1023,7 @@ impl> = - std::sync::OnceLock::new(); - let m = LAST.get_or_init(|| std::sync::Mutex::new(std::time::Instant::now())); - if let Ok(mut last) = m.lock() { - if last.elapsed() >= std::time::Duration::from_secs(2) { - *last = std::time::Instant::now(); - let pend_bytes: usize = self - .pending_batches - .iter() - .flat_map(|b| b.filters().values()) - .map(|f| f.content.len()) - .sum(); - let act_bytes: usize = self - .active_batches - .values() - .flat_map(|b| b.filters().values()) - .map(|f| f.content.len()) - .sum(); - let arc_bytes: usize = - self.active_batches.values().map(|b| b.filters_arc_bytes()).sum(); - let (trk_batches, trk_filters, trk_bytes) = - self.filter_pipeline.buffered_in_trackers(); - tracing::info!( - target: "dash_spv::sync::filters", - "MEM incomplete={} batches ({} filters, {:.0} MB) | pending={} batches ({:.0} MB) | active={} batches (map {:.0} MB + snapshot {:.0} MB) | fheader_cache={} ({:.0} MB)", - trk_batches, - trk_filters, - trk_bytes as f64 / 1e6, - self.pending_batches.len(), - pend_bytes as f64 / 1e6, - self.active_batches.len(), - act_bytes as f64 / 1e6, - arc_bytes as f64 / 1e6, - self.filter_headers.len(), - (self.filter_headers.len() * 36) as f64 / 1e6, - ); - } + let now = std::time::Instant::now(); + let due = self + .last_mem_log + .is_none_or(|last| now.duration_since(last) >= std::time::Duration::from_secs(2)); + if due { + self.last_mem_log = Some(now); + let pend_bytes: usize = self + .pending_batches + .iter() + .flat_map(|b| b.filters().values()) + .map(|f| f.content.len()) + .sum(); + let act_bytes: usize = self + .active_batches + .values() + .flat_map(|b| b.filters().values()) + .map(|f| f.content.len()) + .sum(); + let arc_bytes: usize = + self.active_batches.values().map(|b| b.filters_arc_bytes()).sum(); + let (trk_batches, trk_filters, trk_bytes) = + self.filter_pipeline.buffered_in_trackers(); + tracing::info!( + target: "dash_spv::sync::filters", + "MEM incomplete={} batches ({} filters, {:.0} MB) | pending={} batches ({:.0} MB) | active={} batches (map {:.0} MB + snapshot {:.0} MB) | fheader_cache={} ({:.0} MB)", + trk_batches, + trk_filters, + trk_bytes as f64 / 1e6, + self.pending_batches.len(), + pend_bytes as f64 / 1e6, + self.active_batches.len(), + act_bytes as f64 / 1e6, + arc_bytes as f64 / 1e6, + self.filter_headers.len(), + (self.filter_headers.len() * 36) as f64 / 1e6, + ); } } @@ -802,12 +1308,22 @@ impl= self.progress.filter_header_tip_height() && self.progress.committed_height() >= self.progress.target_height() { - if self.state() == SyncState::Syncing { + if self.state() != SyncState::Synced { self.set_state(SyncState::Synced); } tracing::info!("Filter sync complete at height {}", self.progress.committed_height()); @@ -965,6 +1481,7 @@ impl { + self.note_pending_block(&key, &wallets); blocks_needed.insert(key, wallets); if let Some(b) = self.active_batches.get_mut(&batch_start) { b.set_pending_blocks(b.pending_blocks() + 1); @@ -973,6 +1490,7 @@ impl { + self.note_pending_block(&key, &wallets); blocks_needed.insert(key, wallets); } BlockTrackResult::AlreadyProcessed => {} @@ -1090,7 +1608,7 @@ impl = Vec::new(); @@ -1107,19 +1625,26 @@ impl)> = Vec::with_capacity(starts.len()); + let mut synced_now: BTreeMap = BTreeMap::new(); for &s in &starts { let end = self.active_batches.get(&s).map(|b| b.end_height()).unwrap_or(0); - per_batch.push((s, wallet.wallets_behind(end))); + let behind = wallet.wallets_behind(end); + for id in &behind { + synced_now.entry(*id).or_insert_with(|| wallet.wallet_synced_height(id)); + } + per_batch.push((s, behind)); } - (states, per_batch) + (states, per_batch, synced_now) }; let mut arcs: Vec>> = Vec::with_capacity(starts.len()); + let mut scanned_frontiers: Vec<(u32, BTreeSet)> = Vec::new(); for (s, behind) in behind_by_batch { if let Some(b) = self.active_batches.get_mut(&s) { b.mark_scanned(); - b.set_scanned_wallets(behind); + b.set_scanned_wallets(behind.clone()); + scanned_frontiers.push((b.end_height(), behind)); let arc = b.filters_arc(); if !arc.is_empty() { arcs.push(arc); @@ -1202,6 +1727,7 @@ impl { + self.note_pending_block(&key, &wallets); blocks_needed.insert(key, wallets); if let Some(b) = self.active_batches.get_mut(&owner) { b.set_pending_blocks(b.pending_blocks() + 1); @@ -1210,6 +1736,7 @@ impl { + self.note_pending_block(&key, &wallets); blocks_needed.insert(key, wallets); } BlockTrackResult::AlreadyProcessed => {} @@ -1222,6 +1749,22 @@ impl = wallet_states + .iter() + .map(|s| (s.id, (s.scripts.len() + s.elements.len()) as u32)) + .collect(); + for (end, wallets) in scanned_frontiers { + self.advance_frontier(end, &wallets, &synced_now, &script_counts); + } + crate::timer::P_SCAN.add(t_scan.elapsed()); Ok(events) } @@ -1308,6 +1851,8 @@ impl = + behind.iter().map(|id| (*id, wallet.wallet_synced_height(id))).collect(); let mut wallet_states: Vec = Vec::new(); for wallet_id in &behind { let synced = wallet.wallet_synced_height(wallet_id); @@ -1335,9 +1880,21 @@ impl = behind.clone(); if let Some(batch) = self.active_batches.get_mut(&batch_start) { - batch.set_scanned_wallets(scanned_wallets); + batch.set_scanned_wallets(scanned_wallets.clone()); } + // These wallets are now covered up to `batch_end`. Moving the frontier + // here (rather than at commit) is what lets a restart resume instead of + // re-matching this range; any block this scan matches is added to + // `pending` below, and both go to disk in the same atomic write. + // A wallet with no monitored scripts counts as 0 — nothing derivable can + // later match below it, so it never needs a script-growth rescan. + let script_counts: BTreeMap = wallet_states + .iter() + .map(|s| (s.id, (s.scripts.len() + s.elements.len()) as u32)) + .collect(); + self.advance_frontier(batch_end, &scanned_wallets, &synced_now, &script_counts); + if filters_empty { tracing::debug!("scan_batch: batch filters are empty, returning early"); return Ok(events); @@ -1450,12 +2007,14 @@ impl { + self.note_pending_block(&key, &wallets); blocks_needed.insert(key, wallets); new_blocks_count += 1; } BlockTrackResult::InFlight { wallets, } => { + self.note_pending_block(&key, &wallets); blocks_needed.insert(key, wallets); } BlockTrackResult::AlreadyProcessed => {} @@ -1557,3 +2116,92 @@ impl WalletId { + [b; 32] + } + + /// A wallet monitoring more scripts than its frontier was scanned against is + /// rolled back to birth (so the range below the frontier is rescanned against + /// the grown set); a wallet whose count is unchanged is left alone (fast resume + /// preserved — no rescan starvation). + #[test] + fn rescan_grown_scripts_rolls_back_only_on_growth() { + let mut scan = ScanState::default(); + let a = wid(1); + let b = wid(2); + scan.watermarks.insert( + a, + WalletFrontier { + scanned: 10_000, + synced_at: 10_000, + scanned_scripts: 100, + }, + ); + scan.watermarks.insert( + b, + WalletFrontier { + scanned: 5_000, + synced_at: 5_000, + scanned_scripts: 50, + }, + ); + + let counts = BTreeMap::from([(a, 120), (b, 50)]); + let rolled = scan.rescan_grown_scripts(&counts, 200); + + assert_eq!(rolled, BTreeSet::from([a]), "only the grown wallet rolls back"); + assert_eq!(scan.watermarks[&a].scanned, 200, "grown wallet rescans from birth"); + assert_eq!(scan.watermarks[&b].scanned, 5_000, "unchanged wallet untouched"); + + // Idempotent once the counts match what is recorded: no repeat rollback. + assert!(scan.rescan_grown_scripts(&BTreeMap::from([(a, 120), (b, 50)]), 200).is_empty()); + } + + /// A frontier already at or below birth is never rolled "back" past it, and a + /// wallet absent from the counts map (no monitored scripts) never rescans. + #[test] + fn rescan_grown_scripts_respects_birth_and_missing() { + let mut scan = ScanState::default(); + let a = wid(1); + scan.watermarks.insert( + a, + WalletFrontier { + scanned: 150, + synced_at: 150, + scanned_scripts: 10, + }, + ); + // birth 200 is above `scanned` 150 → nothing to roll back. + assert!(scan.rescan_grown_scripts(&BTreeMap::from([(a, 999)]), 200).is_empty()); + assert_eq!(scan.watermarks[&a].scanned, 150); + // Empty counts → current count 0, never exceeds recorded → no rollback. + assert!(scan.rescan_grown_scripts(&BTreeMap::new(), 0).is_empty()); + } + + /// `scanned_scripts` survives the encode/decode round-trip that persists the + /// frontier, so the growth check works across a restart. + #[test] + fn encode_decode_roundtrips_scanned_scripts() { + let mut scan = ScanState::default(); + scan.watermarks.insert( + wid(7), + WalletFrontier { + scanned: 12_345, + synced_at: 12_000, + scanned_scripts: 777, + }, + ); + let decoded = ScanState::decode(&scan.encode()).expect("roundtrip decode"); + let f = decoded.watermarks[&wid(7)]; + assert_eq!(f.scanned, 12_345); + assert_eq!(f.synced_at, 12_000); + assert_eq!(f.scanned_scripts, 777); + } +} diff --git a/dash-spv/src/sync/filters/pipeline.rs b/dash-spv/src/sync/filters/pipeline.rs index 014cd8f5f..d77ee1076 100644 --- a/dash-spv/src/sync/filters/pipeline.rs +++ b/dash-spv/src/sync/filters/pipeline.rs @@ -15,7 +15,7 @@ use dashcore::network::message_filter::GetCFilters; use dashcore::BlockHash; use crate::error::{SyncError, SyncResult}; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::BlockHeaderStorage; use crate::sync::download_coordinator::DownloadCoordinator; use crate::sync::filters::batch::FiltersBatch; diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 1d651f3ac..2ddb9990a 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::{SyncError, SyncResult}; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::{BlockHeaderStorage, FilterHeaderStorage, FilterStorage}; use crate::sync::sync_manager::ensure_not_started; use crate::sync::{ @@ -35,12 +35,12 @@ impl< self.progress.update_target_height(height); } - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { - &[crate::network2::MessageType::CFilter] + fn subscribed_commands(&self) -> &'static [crate::network::MessageType] { + &[crate::network::MessageType::CFilter] } - fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { - if let crate::network2::RequestKey::CFilters(start_height) = key { + fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { + if let crate::network::RequestKey::CFilters(start_height) = key { self.filter_pipeline.mark_on_flight(*start_height); } } @@ -52,6 +52,12 @@ impl< /// network-manager retry layer. fn on_disconnect(&mut self) {} + /// Flush the scan frontier on a graceful stop, so a restart resumes where the + /// scan left off instead of re-matching from the wallets' `synced_height`. + async fn on_shutdown(&mut self) { + self.persist_scan_state().await; + } + async fn start_sync( &mut self, network: &Arc, @@ -195,6 +201,12 @@ impl< // `tracker.track` residual. self.tracker.record_processed(*height, *block_hash, wallets); + // The scan is no longer owed this block, so a restart must not + // re-request it. Cleared before `try_process_batch` below, which + // may commit the batch and move the wallets' `synced_height` past + // this height. + self.clear_pending_block(*height); + // Check if this block is part of our tracked blocks if let Some((_, batch_start)) = self.tracker.finish_in_flight(block_hash) { if let Some(batch) = self.active_batches.get_mut(&batch_start) { @@ -216,6 +228,15 @@ impl< if let Some(batch) = self.active_batches.get_mut(&batch_start) { batch.add_scripts_for_wallet(*wallet_id, scripts.iter().cloned()); } + // `add_scripts_for_wallet` only re-matches OPEN batches. Ranges + // already committed below the frontier are closed, so roll this + // wallet's frontier back to its birth to force a durable rescan + // of them against the new scripts on the next pass — otherwise a + // transaction to a freshly-derived address in a processed block + // is skipped for good. Uses the cached birth (NOT a wallet read): + // reading the wallet here deadlocks against block processing under + // rapid restart. + self.note_new_scripts(wallet_id, self.birth_height); } return self.try_process_batch().await; @@ -229,6 +250,11 @@ impl< } async fn tick(&mut self, network: &Arc) -> SyncResult> { + // Coalesce scan-frontier writes here: hot-path mutators only mark it dirty, + // and this flushes at most once per tick (a no-op when nothing changed). + // Bounds staleness to one tick; a hard kill in that window just rescans. + self.persist_scan_state().await; + // Detect a wallet that was added behind our scan progress and rescan // from its `synced_height`. Reset committed_height to the lowest // synced_height across the stale wallets only, so already-synced diff --git a/dash-spv/src/sync/instantsend/sync_manager.rs b/dash-spv/src/sync/instantsend/sync_manager.rs index 6abc5e1b5..14413add8 100644 --- a/dash-spv/src/sync/instantsend/sync_manager.rs +++ b/dash-spv/src/sync/instantsend/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::sync::{ InstantSendManager, ManagerIdentifier, SyncEvent, SyncManager, SyncManagerProgress, SyncState, }; @@ -23,8 +23,8 @@ impl SyncManager for InstantSendManager { self.progress.set_state(state); } - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { - use crate::network2::MessageType; + fn subscribed_commands(&self) -> &'static [crate::network::MessageType] { + use crate::network::MessageType; &[MessageType::IsDLock, MessageType::Inv] } @@ -34,7 +34,7 @@ impl SyncManager for InstantSendManager { async fn handle_message( &mut self, - peer: SocketAddr, + _peer: SocketAddr, msg: NetworkMessage, network: &Arc, ) -> SyncResult> { diff --git a/dash-spv/src/sync/masternodes/manager.rs b/dash-spv/src/sync/masternodes/manager.rs index c8003977d..78cef2ca7 100644 --- a/dash-spv/src/sync/masternodes/manager.rs +++ b/dash-spv/src/sync/masternodes/manager.rs @@ -13,7 +13,7 @@ use tokio::sync::RwLock; use super::pipeline::MnListDiffPipeline; use crate::error::{SyncError, SyncResult}; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::BlockHeaderStorage; use crate::sync::{MasternodesProgress, SyncEvent, SyncManager, SyncState}; use dashcore::network::message::NetworkMessage; diff --git a/dash-spv/src/sync/masternodes/pipeline.rs b/dash-spv/src/sync/masternodes/pipeline.rs index 7d21176ce..0b4752700 100644 --- a/dash-spv/src/sync/masternodes/pipeline.rs +++ b/dash-spv/src/sync/masternodes/pipeline.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::sync::Arc; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::sync::download_coordinator::DownloadCoordinator; use dashcore::network::message::NetworkMessage; use dashcore::network::message_sml::{GetMnListDiff, MnListDiff}; diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index fbfb9672c..2198b161d 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -1,6 +1,6 @@ use super::manager::PipelineMode; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::BlockHeaderStorage; use crate::sync::{ ManagerIdentifier, MasternodesManager, SyncEvent, SyncManager, SyncManagerProgress, SyncState, @@ -223,13 +223,13 @@ impl SyncManager for MasternodesManager { self.progress.update_target_height(height); } - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { - use crate::network2::MessageType; + fn subscribed_commands(&self) -> &'static [crate::network::MessageType] { + use crate::network::MessageType; &[MessageType::MnListDiff, MessageType::QrInfo] } - fn mark_on_flight(&mut self, key: &crate::network2::RequestKey) { - if let crate::network2::RequestKey::MnListDiff(target_hash) = key { + fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { + if let crate::network::RequestKey::MnListDiff(target_hash) = key { self.sync_state.mnlistdiff_pipeline.mark_on_flight(target_hash); } } diff --git a/dash-spv/src/sync/mempool/manager.rs b/dash-spv/src/sync/mempool/manager.rs index 3741acbce..f4df21dfb 100644 --- a/dash-spv/src/sync/mempool/manager.rs +++ b/dash-spv/src/sync/mempool/manager.rs @@ -21,7 +21,7 @@ use super::filter::build_wallet_bloom_filter; use super::BLOOM_FALSE_POSITIVE_RATE; use crate::client::config::MempoolStrategy; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::sync::mempool::MempoolProgress; use crate::sync::SyncEvent; use crate::types::UnconfirmedTransaction; @@ -110,21 +110,30 @@ impl MempoolManager { ) -> SyncResult<()> { tracing::info!("Activating mempool on peer {} (strategy: {:?})", peer, self.strategy); + // Addressed to THIS peer, not handed to the router: relay is per-peer state on the + // remote node, so a `filterclear`/`filterload` that lands on a different peer than + // the one we mean to activate simply leaves this one mute. match self.strategy { MempoolStrategy::BloomFilter => { self.load_bloom_filter(peer, network).await?; } MempoolStrategy::FetchAll => { - network.send(NetworkMessage::FilterClear).await; + network.send_to(peer, NetworkMessage::FilterClear).await; } } - network.send(NetworkMessage::MemPool).await; + network.send_to(peer, NetworkMessage::MemPool).await; self.peers.insert(peer, Some(VecDeque::new())); Ok(()) } /// Activate mempool relay on all connected but not-yet-activated peers. + /// + /// `self.peers` is seeded by `PeerConnected`, which only reaches us because the + /// network manager connects in `start` — after the coordinator has spawned and + /// subscribed us. It used to connect in `new`, so those events fired into the void, + /// this map stayed empty, no `filterload` ever went out, and peers (we handshake with + /// `relay=false`) never announced a transaction or an InstantSend lock to us. pub(super) async fn activate_all_peers( &mut self, network: &Arc, @@ -168,7 +177,7 @@ impl MempoolManager { filter_load.filter.len() ); - network.send(NetworkMessage::FilterLoad(filter_load)).await; + network.send_to(peer, NetworkMessage::FilterLoad(filter_load)).await; Ok(()) } @@ -190,9 +199,9 @@ impl MempoolManager { } for peer in activated { - network.send(NetworkMessage::FilterClear).await; + network.send_to(peer, NetworkMessage::FilterClear).await; self.load_bloom_filter(peer, network).await?; - network.send(NetworkMessage::MemPool).await; + network.send_to(peer, NetworkMessage::MemPool).await; } Ok(()) @@ -303,7 +312,10 @@ impl MempoolManager { peer, total_queued, ); - network.send(NetworkMessage::GetData(inventory)).await; + // Ask the peer that ANNOUNCED these txids, not whichever the router favours: + // a mempool transaction only exists on the nodes that have it, and the queue + // was built per-peer precisely so each one is asked for what it offered. + network.send_to(peer, NetworkMessage::GetData(inventory)).await; } Ok(()) } diff --git a/dash-spv/src/sync/mempool/sync_manager.rs b/dash-spv/src/sync/mempool/sync_manager.rs index 47ec1a406..b48ce790a 100644 --- a/dash-spv/src/sync/mempool/sync_manager.rs +++ b/dash-spv/src/sync/mempool/sync_manager.rs @@ -1,6 +1,6 @@ use super::manager::MEMPOOL_TX_EXPIRY; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::{NetworkEvent, PeerNetworkManager}; use crate::sync::{ ManagerIdentifier, MempoolManager, SyncEvent, SyncManager, SyncManagerProgress, SyncState, }; @@ -24,8 +24,8 @@ impl SyncManager for MempoolManager { self.progress.set_state(state); } - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType] { - use crate::network2::MessageType; + fn subscribed_commands(&self) -> &'static [crate::network::MessageType] { + use crate::network::MessageType; &[MessageType::Inv, MessageType::Tx] } @@ -45,6 +45,39 @@ impl SyncManager for MempoolManager { Ok(vec![]) } + /// Track the peer set as the network reports it. + /// + /// This is the only thing that seeds `self.peers`, and it works only because the + /// network manager connects in `start` — after the coordinator has subscribed us. + /// Everything else here (relay activation, and so every transaction and InstantSend + /// lock we ever see) hangs off it. + async fn handle_network_event( + &mut self, + event: &NetworkEvent, + network: &Arc, + ) -> SyncResult> { + match event { + NetworkEvent::PeerConnected(addr) => { + self.handle_peer_connected(*addr); + // Filters already synced (a peer joining mid-run): enable relay on it now. + // Otherwise `FiltersSyncComplete` will. + if self.state() == SyncState::Synced { + self.activate_peer(*addr, network).await?; + } + Ok(vec![]) + } + NetworkEvent::PeerDisconnected(addr) => { + // Hands this peer's queued txids to another activated one rather than + // dropping them. + self.handle_peer_disconnected(*addr); + Ok(vec![]) + } + _ => { + crate::sync::sync_manager::default_handle_network_event(self, event, network).await + } + } + } + fn on_disconnect(&mut self) { self.clear_pending(); } diff --git a/dash-spv/src/sync/sync_coordinator.rs b/dash-spv/src/sync/sync_coordinator.rs index 7b83c8a95..2476ae6c9 100644 --- a/dash-spv/src/sync/sync_coordinator.rs +++ b/dash-spv/src/sync/sync_coordinator.rs @@ -13,7 +13,7 @@ use tokio_stream::wrappers::WatchStream; use tokio_util::sync::CancellationToken; use crate::error::SyncResult; -use crate::network2::PeerNetworkManager; +use crate::network::PeerNetworkManager; use crate::storage::{ BlockHeaderStorage, BlockStorage, FilterHeaderStorage, FilterStorage, MetadataStorage, }; @@ -70,7 +70,7 @@ where pub block_headers: Option>, pub filter_headers: Option>, pub filters: Option>, - pub blocks: Option>, + pub blocks: Option>, pub masternode: Option>, pub chainlock: Option>, pub instantsend: Option, diff --git a/dash-spv/src/sync/sync_manager.rs b/dash-spv/src/sync/sync_manager.rs index 340ce1474..d0fd92d46 100644 --- a/dash-spv/src/sync/sync_manager.rs +++ b/dash-spv/src/sync/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network2::{NetworkEvent, PeerNetworkManager, RequestKey}; +use crate::network::{NetworkEvent, PeerNetworkManager, RequestKey}; use crate::sync::{ BlockHeadersProgress, BlocksProgress, ChainLockProgress, FilterHeadersProgress, FiltersProgress, InstantSendProgress, ManagerIdentifier, MasternodesProgress, MempoolProgress, @@ -73,12 +73,15 @@ impl SyncManagerTaskContext { } } -// Display for the network2 event so the sync loop / broadcast monitor can log -// it (they require `Display`). Kept here to avoid modifying the network2 module. +// Display for the network event so the sync loop / broadcast monitor can log +// it (they require `Display`). Kept here to avoid modifying the network module. impl std::fmt::Display for NetworkEvent { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - NetworkEvent::PeersUpdated => write!(f, "PeersUpdated"), + NetworkEvent::PeersUpdated { + connected_count, + .. + } => write!(f, "PeersUpdated({connected_count} peers)"), NetworkEvent::PeerConnected(addr) => write!(f, "PeerConnected({addr})"), NetworkEvent::PeerDisconnected(addr) => write!(f, "PeerDisconnected({addr})"), NetworkEvent::RequestOnFlight(_) => write!(f, "RequestOnFlight"), @@ -86,6 +89,36 @@ impl std::fmt::Display for NetworkEvent { } } +/// The default `SyncManager::handle_network_event` body, callable from an override. +pub(super) async fn default_handle_network_event( + manager: &mut M, + event: &NetworkEvent, + network: &Arc, +) -> SyncResult> { + // `PeersUpdated` is the cue to kick off the initial requests: the network manager + // connects in `start`, after every manager has subscribed, so this is the first thing + // we hear from it. Individual peer disconnects are recovered by per-request + // timeout+retry, so they don't stop sync. + if let NetworkEvent::PeersUpdated { + .. + } = event + { + // Seed every manager's target from the peers' advertised tip so the + // height shows up right away (matches the pre-network `best_height`). + manager.update_target_height(network.tip()); + if manager.state() == SyncState::WaitingForConnections { + tracing::info!("{} - peers available, starting sync", manager.identifier()); + return manager.start_sync(network).await; + } + } + // A request just went on the wire: let the owning pipeline start its + // response timeout from now (see `mark_on_flight`). + if let NetworkEvent::RequestOnFlight(key) = event { + manager.mark_on_flight(key); + } + Ok(vec![]) +} + /// Guard that verifies a manager has not already been started. pub(super) fn ensure_not_started( state: SyncState, @@ -116,7 +149,7 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { /// /// The network manager uses this to route only relevant messages to each /// manager's task via topic-based filtering. - fn subscribed_commands(&self) -> &'static [crate::network2::MessageType]; + fn subscribed_commands(&self) -> &'static [crate::network::MessageType]; /// Start the sync process. /// @@ -187,30 +220,15 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { /// Handle a network event (peer connection changes). /// /// Default implementation handles state transitions for WaitingForConnections. - /// Managers can override to customize behavior. + /// Managers can override to customize behavior — an override that only cares about + /// some variants should delegate the rest to [`default_handle_network_event`], since + /// Rust gives no way to call a trait's default body from an override. async fn handle_network_event( &mut self, event: &NetworkEvent, network: &Arc, ) -> SyncResult> { - // Peers are connected before the manager starts; PeersUpdated is our - // cue to kick off the initial requests. Individual peer disconnects are - // recovered by per-request timeout+retry, so they don't stop sync. - if let NetworkEvent::PeersUpdated = event { - // Seed every manager's target from the peers' advertised tip so the - // height shows up right away (matches the pre-network2 `best_height`). - self.update_target_height(network.tip()); - if self.state() == SyncState::WaitingForConnections { - tracing::info!("{} - peers available, starting sync", self.identifier()); - return self.start_sync(network).await; - } - } - // A request just went on the wire: let the owning pipeline start its - // response timeout from now (see `mark_on_flight`). - if let NetworkEvent::RequestOnFlight(key) = event { - self.mark_on_flight(key); - } - Ok(vec![]) + default_handle_network_event(self, event, network).await } /// Start the response timeout for a request the network manager just put on @@ -219,6 +237,11 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { /// `mark_on_flight`. Keeps retry counting from actual send, not from queue. fn mark_on_flight(&mut self, _key: &RequestKey) {} + /// Flush any durable state before the task exits on shutdown. Default no-op; + /// managers holding a write-back cache (the filters manager's scan frontier) + /// override it so a graceful stop leaves that state on disk for the next run. + async fn on_shutdown(&mut self) {} + /// Retrieves the current progress of the Manager. fn progress(&self) -> SyncManagerProgress; @@ -248,20 +271,11 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { // Tick interval for periodic housekeeping let mut tick_interval = interval(Duration::from_millis(100)); - // Peers are already connected when this task starts: network2 connects - // at construction, and its one-shot PeersUpdated may fire before we - // subscribe. So replay it now (seeds the target height and kicks off - // sync) instead of waiting for the event. - { - let progress_before = self.progress(); - match self.handle_network_event(&NetworkEvent::PeersUpdated, &context.network).await { - Ok(events) => { - context.emit_sync_events(events); - self.try_emit_progress(progress_before, &context.progress_sender); - } - Err(e) => tracing::debug!("{} initial start_sync skipped: {}", identifier, e), - } - } + // No bootstrap replay here: the network manager connects in `start`, which the + // client calls only after every manager has been spawned and subscribed, so the + // real `PeersUpdated` reaches us. (This used to synthesize one, because the + // network connected in `new` and its one-shot event fired before anyone was + // listening. Replaying it now would fire with zero peers connected.) tracing::info!("{} task entering main loop", identifier); @@ -269,6 +283,7 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { tokio::select! { _ = context.shutdown.cancelled() => { tracing::info!("{} task received shutdown signal", identifier); + self.on_shutdown().await; break; } // Process incoming network messages diff --git a/dash-spv/src/test_utils/event_handler.rs b/dash-spv/src/test_utils/event_handler.rs index 20651eaad..7a5dd1143 100644 --- a/dash-spv/src/test_utils/event_handler.rs +++ b/dash-spv/src/test_utils/event_handler.rs @@ -6,7 +6,7 @@ use tokio::sync::{broadcast, watch}; use crate::client::EventHandler; -use crate::network2::NetworkEvent; +use crate::network::NetworkEvent; use crate::sync::{SyncEvent, SyncProgress}; use key_wallet_manager::WalletEvent; diff --git a/dash-spv/src/test_utils/mod.rs b/dash-spv/src/test_utils/mod.rs index 61acedda0..943a51e12 100644 --- a/dash-spv/src/test_utils/mod.rs +++ b/dash-spv/src/test_utils/mod.rs @@ -6,7 +6,6 @@ mod event_handler; mod filter; mod fs_helpers; pub(crate) mod masternode_network; -mod network; mod node; mod types; mod wallet; @@ -20,7 +19,6 @@ pub use context::DashdTestContext; pub use event_handler::TestEventHandler; pub use fs_helpers::retain_test_dir; pub use masternode_network::MasternodeTestContext; -pub use network::{test_socket_address, MockNetworkManager}; pub use node::{DashCoreNode, TestChain, WalletFile}; pub use wallet::{ create_test_wallet, default_test_account_options, init_test_logging, diff --git a/dash-spv/src/test_utils/network.rs b/dash-spv/src/test_utils/network.rs deleted file mode 100644 index 5ac883944..000000000 --- a/dash-spv/src/test_utils/network.rs +++ /dev/null @@ -1,193 +0,0 @@ -use crate::error::{NetworkError, NetworkResult}; -use crate::network::peer::Peer; -use crate::network::{ - Message, MessageDispatcher, MessageType, NetworkEvent, NetworkManager, NetworkRequest, - RequestSender, -}; -use async_trait::async_trait; -use dashcore::{ - block::Header as BlockHeader, network::message::NetworkMessage, - network::message_blockdata::GetHeadersMessage, BlockHash, Network, -}; -use dashcore_hashes::Hash; -use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; -use std::time::Duration; -use tokio::sync::broadcast; -use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; -use tokio::sync::Mutex; - -pub fn test_socket_address(id: u8) -> SocketAddr { - SocketAddr::from(([127, 0, 0, id], id as u16)) -} - -/// Mock network manager for testing -pub struct MockNetworkManager { - connected: bool, - connected_peer: SocketAddr, - headers_chain: Vec, - message_dispatcher: Mutex, - sent_messages: Vec, - /// Request sender for outgoing messages. - request_tx: UnboundedSender, - /// Receiver generated in the constructor. Can be taken out of the struct for testing. - request_rx: Option>, - /// Event bus for network events. - network_event_sender: broadcast::Sender, -} - -impl MockNetworkManager { - /// Create a new mock network manager - pub fn new() -> Self { - let (request_tx, request_rx) = unbounded_channel(); - Self { - connected: true, - connected_peer: SocketAddr::new(std::net::Ipv4Addr::LOCALHOST.into(), 9999), - headers_chain: Vec::new(), - message_dispatcher: Mutex::new(MessageDispatcher::default()), - sent_messages: Vec::new(), - request_tx, - request_rx: Some(request_rx), - network_event_sender: broadcast::Sender::new(100000), - } - } - - pub fn take_receiver(&mut self) -> Option> { - self.request_rx.take() - } - - /// Add a chain of headers for testing - pub fn add_headers_chain(&mut self, genesis_hash: BlockHash, count: usize) { - let mut headers = Vec::new(); - let mut prev_hash = genesis_hash; - - // Skip genesis (height 0) as it's already in the storage - for i in 1..count { - let header = BlockHeader { - version: dashcore::block::Version::from_consensus(1), - prev_blockhash: prev_hash, - merkle_root: dashcore::hashes::sha256d::Hash::all_zeros().into(), - time: 1000000 + i as u32, - bits: dashcore::CompactTarget::from_consensus(0x207fffff), - nonce: i as u32, - }; - - prev_hash = header.block_hash(); - headers.push(header); - } - - self.headers_chain = headers; - } - - /// Process GetHeaders request and return appropriate headers - fn process_getheaders(&self, msg: &GetHeadersMessage) -> Vec { - // Find the starting point in our chain - let start_idx = if msg.locator_hashes.is_empty() { - 0 - } else { - // Find the first locator hash we recognize - let mut found_idx = None; - for locator in &msg.locator_hashes { - for (idx, header) in self.headers_chain.iter().enumerate() { - if header.block_hash() == *locator { - found_idx = Some(idx + 1); // Start from next header - break; - } - } - if found_idx.is_some() { - break; - } - } - found_idx.unwrap_or(0) - }; - - // Return up to 2000 headers starting from start_idx - let end_idx = (start_idx + 2000).min(self.headers_chain.len()); - - if start_idx < self.headers_chain.len() { - self.headers_chain[start_idx..end_idx].to_vec() - } else { - Vec::new() - } - } - - pub fn sent_messages(&self) -> &Vec { - &self.sent_messages - } -} - -impl Default for MockNetworkManager { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl NetworkManager for MockNetworkManager { - async fn message_receiver(&mut self, types: &[MessageType]) -> UnboundedReceiver { - self.message_dispatcher.lock().await.message_receiver(types) - } - - fn request_sender(&self) -> RequestSender { - RequestSender::new(self.request_tx.clone()) - } - - async fn connect(&mut self) -> NetworkResult<()> { - self.connected = true; - Ok(()) - } - - async fn disconnect(&mut self) -> NetworkResult<()> { - self.connected = false; - Ok(()) - } - - async fn send_message(&mut self, message: NetworkMessage) -> NetworkResult<()> { - if !self.connected { - return Err(NetworkError::NotConnected); - } - - // Process GetHeaders requests - if let NetworkMessage::GetHeaders(ref getheaders) = message { - let headers = self.process_getheaders(getheaders); - if !headers.is_empty() { - let msg = Message::new(self.connected_peer, NetworkMessage::Headers(headers)); - self.message_dispatcher.lock().await.dispatch(&msg); - } - } - - self.sent_messages.push(message); - - Ok(()) - } - fn peer_count(&self) -> usize { - if self.connected { - 1 - } else { - 0 - } - } - - async fn broadcast(&self, _message: NetworkMessage) -> NetworkResult<()> { - panic!("Broadcast not implemented for MockNetworkManager"); - } - - async fn dispatch_local(&self, message: NetworkMessage) { - let local_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0)); - let msg = Message::new(local_addr, message); - self.message_dispatcher.lock().await.dispatch(&msg); - } - - async fn disconnect_peer(&self, _addr: &SocketAddr, _reason: &str) -> NetworkResult<()> { - panic!("Disconnect peer not implemented for MockNetworkManager"); - } - - fn subscribe_network_events(&self) -> broadcast::Receiver { - self.network_event_sender.subscribe() - } -} - -impl Peer { - pub fn dummy(addr: SocketAddr) -> Self { - Peer::new(addr, Duration::from_secs(10), Network::Mainnet) - } -} diff --git a/dash-spv/src/timer.rs b/dash-spv/src/timer.rs index 5cf9d179c..5f4f74a8f 100644 --- a/dash-spv/src/timer.rs +++ b/dash-spv/src/timer.rs @@ -89,7 +89,7 @@ pub static P_COMMIT: Prof = Prof::new("filt.commit_loop"); // If per-call time balloons with more peers, that Mutex is the ~70k/s serialization point. pub static P_NET_DISPATCH: Prof = Prof::new("net.dispatch(lock)"); -// network2 pump (single task draining PeerEvents): BUSY = per-message dispatch work +// network pump (single task draining PeerEvents): BUSY = per-message dispatch work // (subscriber lock + clone + channel send); IDLE = time blocked in inbound.recv().await. // IDLE >> BUSY => the pump is starved (readers/peers slow), not the bottleneck. pub static P_PUMP_BUSY: Prof = Prof::new("pump.dispatch_busy"); diff --git a/dash-spv/tests/dashd_masternode/setup.rs b/dash-spv/tests/dashd_masternode/setup.rs index 773dd5c07..fbab71cfc 100644 --- a/dash-spv/tests/dashd_masternode/setup.rs +++ b/dash-spv/tests/dashd_masternode/setup.rs @@ -2,14 +2,12 @@ use std::net::SocketAddr; use std::time::{Duration, Instant}; use dash_spv::error::Result as SpvResult; -use dash_spv::network::NetworkEvent; use dash_spv::test_utils::{ create_test_wallet, init_test_logging, next_unused_receive_address, retain_test_dir, MasternodeTestContext, TestEventHandler, }; use dash_spv::{ client::{ClientConfig, DashSpvClient}, - network::PeerNetworkManager, storage::DiskStorageManager, sync::{SyncEvent, SyncProgress}, LoggingGuard, Network, @@ -27,8 +25,7 @@ use tokio::time; /// Timeout for masternode sync tests (masternode sync takes longer than wallet sync). pub(super) const SYNC_TIMEOUT: u64 = 60; -pub(super) type TestClient = - DashSpvClient, PeerNetworkManager, DiskStorageManager>; +pub(super) type TestClient = DashSpvClient, DiskStorageManager>; pub(super) struct ClientHandle { pub(super) client: TestClient, @@ -36,7 +33,6 @@ pub(super) struct ClientHandle { pub(super) progress_receiver: watch::Receiver, pub(super) sync_event_receiver: broadcast::Receiver, pub(super) wallet_event_receiver: broadcast::Receiver, - pub(super) _network_event_receiver: broadcast::Receiver, pub(super) engine: Arc>, } @@ -165,8 +161,6 @@ pub(super) async fn create_and_start_client( config: &ClientConfig, wallet: Arc>>, ) -> ClientHandle { - let network_manager = - PeerNetworkManager::new(config).await.expect("Failed to create network manager"); let storage_manager = DiskStorageManager::new(config).await.expect("Failed to create storage manager"); @@ -174,12 +168,10 @@ pub(super) async fn create_and_start_client( let progress_receiver = handler.subscribe_progress(); let sync_event_receiver = handler.subscribe_sync_events(); let wallet_event_receiver = handler.subscribe_wallet_events(); - let _network_event_receiver = handler.subscribe_network_events(); - let client = - DashSpvClient::new(config.clone(), network_manager, storage_manager, wallet, vec![handler]) - .await - .expect("Failed to create client"); + let client = DashSpvClient::new(config.clone(), storage_manager, wallet, vec![handler]) + .await + .expect("Failed to create client"); let engine = client.masternode_list_engine().expect("Engine should be initialized after creation"); @@ -193,7 +185,6 @@ pub(super) async fn create_and_start_client( progress_receiver, sync_event_receiver, wallet_event_receiver, - _network_event_receiver, engine, } } diff --git a/dash-spv/tests/dashd_sync/helpers.rs b/dash-spv/tests/dashd_sync/helpers.rs index a3be238f3..7a482d748 100644 --- a/dash-spv/tests/dashd_sync/helpers.rs +++ b/dash-spv/tests/dashd_sync/helpers.rs @@ -220,6 +220,45 @@ pub(super) async fn wait_for_mempool_tx( } } +/// Wait for a *specific* transaction to appear in the mempool, ignoring unrelated +/// mempool events. +/// +/// Unlike [`wait_for_mempool_tx`], which returns the first mempool +/// `TransactionDetected` it sees, this skips events whose txid isn't `expected`. +/// A test that funds an address and then broadcasts its own transaction can have +/// a still-in-flight funding tx surface as a mempool event first (the wallet +/// legitimately reports it); matching on the exact txid keeps that stale event +/// from being mistaken for the transaction under test. Returns `true` once +/// `expected` is observed, `false` on timeout. +pub(super) async fn wait_for_expected_mempool_tx( + receiver: &mut broadcast::Receiver, + expected: Txid, + max_wait: Duration, +) -> bool { + let timeout = tokio::time::sleep(max_wait); + tokio::pin!(timeout); + + loop { + tokio::select! { + _ = &mut timeout => return false, + result = receiver.recv() => { + match result { + Ok(WalletEvent::TransactionDetected { ref record, .. }) + if matches!( + record.context, + TransactionContext::Mempool | TransactionContext::InstantSend(_) + ) && record.txid == expected => + { + return true; + } + Ok(_) => continue, + Err(_) => return false, + } + } + } + } +} + /// Wait for the mempool manager to reach `Synced` state via the progress watch channel. /// Returns `true` if the state is reached within the timeout, `false` otherwise. pub(super) async fn wait_for_mempool_synced( diff --git a/dash-spv/tests/dashd_sync/setup.rs b/dash-spv/tests/dashd_sync/setup.rs index 27c8b5415..b328ae906 100644 --- a/dash-spv/tests/dashd_sync/setup.rs +++ b/dash-spv/tests/dashd_sync/setup.rs @@ -7,7 +7,6 @@ use dash_spv::test_utils::{ }; use dash_spv::{ client::{ClientConfig, DashSpvClient}, - network::PeerNetworkManager, storage::DiskStorageManager, sync::{ProgressPercentage, SyncEvent, SyncProgress}, LoggingGuard, Network, @@ -226,8 +225,7 @@ impl Drop for TestContext { } /// Type alias for the SPV client used in tests. -pub(super) type TestClient = - DashSpvClient, PeerNetworkManager, DiskStorageManager>; +pub(super) type TestClient = DashSpvClient, DiskStorageManager>; /// A `ClientHandle` is a utility structure that manages the state and handles for a `TestClient` /// required to interact with the synchronization process and event channels. @@ -281,8 +279,6 @@ pub(super) async fn create_and_start_client( config: &ClientConfig, wallet: Arc>>, ) -> ClientHandle { - let network_manager = - PeerNetworkManager::new(config).await.expect("Failed to create network manager"); let storage_manager = DiskStorageManager::new(config).await.expect("Failed to create storage manager"); @@ -291,10 +287,9 @@ pub(super) async fn create_and_start_client( let sync_event_receiver = handler.subscribe_sync_events(); let network_event_receiver = handler.subscribe_network_events(); - let client = - DashSpvClient::new(config.clone(), network_manager, storage_manager, wallet, vec![handler]) - .await - .expect("Failed to create client"); + let client = DashSpvClient::new(config.clone(), storage_manager, wallet, vec![handler]) + .await + .expect("Failed to create client"); let wallet_event_receiver = { let w = client.wallet().read().await; diff --git a/dash-spv/tests/dashd_sync/tests_mempool.rs b/dash-spv/tests/dashd_sync/tests_mempool.rs index 29c407dd5..f7ef7b75c 100644 --- a/dash-spv/tests/dashd_sync/tests_mempool.rs +++ b/dash-spv/tests/dashd_sync/tests_mempool.rs @@ -410,12 +410,12 @@ async fn test_mempool_peer_disconnect_reactivation() { let (fa_disc, bf_disc) = tokio::join!( wait_for_network_event( &mut fa_net_rx, - |e| matches!(e, NetworkEvent::PeerDisconnected { address } if *address == ctx.dashd.addr), + |e| matches!(e, NetworkEvent::PeerDisconnected(address) if *address == ctx.dashd.addr), Duration::from_secs(10), ), wait_for_network_event( &mut bf_net_rx, - |e| matches!(e, NetworkEvent::PeerDisconnected { address } if *address == ctx.dashd.addr), + |e| matches!(e, NetworkEvent::PeerDisconnected(address) if *address == ctx.dashd.addr), Duration::from_secs(10), ), ); @@ -449,10 +449,10 @@ async fn test_mempool_peer_disconnect_reactivation() { _ = &mut deadline => panic!("{}: timed out waiting for both peer disconnects", label), result = receiver.recv() => { match result { - Ok(NetworkEvent::PeerDisconnected { address }) if address == ctx.dashd.addr => { + Ok(NetworkEvent::PeerDisconnected(address)) if address == ctx.dashd.addr => { seen_dashd1 = true; } - Ok(NetworkEvent::PeerDisconnected { address }) if address == dashd2.addr => { + Ok(NetworkEvent::PeerDisconnected(address)) if address == dashd2.addr => { seen_dashd2 = true; } _ => {} diff --git a/dash-spv/tests/dashd_sync/tests_multi_wallet.rs b/dash-spv/tests/dashd_sync/tests_multi_wallet.rs index 276f43cbf..2dc20d57b 100644 --- a/dash-spv/tests/dashd_sync/tests_multi_wallet.rs +++ b/dash-spv/tests/dashd_sync/tests_multi_wallet.rs @@ -522,72 +522,14 @@ async fn test_runtime_add_with_tip_advance_during_rescan() { .expect("add W2 at runtime") }; - // Wait for the rescan trigger to lower committed_height, then for it to - // climb back to the midpoint without yet reaching tip. Polling raw - // committed_height isn't enough because immediately after W2 is added the - // FiltersManager's tick has not yet fired, so committed_height is still - // at the post-W1-sync value (the chain tip). - let midpoint = initial_height / 2; - let trigger_deadline = tokio::time::Instant::now() + Duration::from_secs(30); - loop { - let filter_height = { - let progress = client_handle.progress_receiver.borrow_and_update(); - progress.filters().ok().map(|f| f.committed_height()).unwrap_or(0) - }; - if filter_height < initial_height { - tracing::info!( - "Rescan trigger fired: committed_height dropped to {} (was {})", - filter_height, - initial_height, - ); - break; - } - if tokio::time::Instant::now() > trigger_deadline { - panic!( - "rescan trigger did not fire within 30s, committed_height still at {}", - filter_height, - ); - } - tokio::select! { - _ = client_handle.progress_receiver.changed() => {} - _ = tokio::time::sleep(Duration::from_millis(20)) => {} - } - } - - let inflight_deadline = tokio::time::Instant::now() + Duration::from_secs(60); - loop { - let filter_height = { - let progress = client_handle.progress_receiver.borrow_and_update(); - progress.filters().ok().map(|f| f.committed_height()).unwrap_or(0) - }; - if filter_height >= midpoint && filter_height < initial_height { - tracing::info!( - "Mid-rescan: filter committed_height={} (midpoint={}, tip={})", - filter_height, - midpoint, - initial_height, - ); - break; - } - if filter_height >= initial_height { - panic!( - "rescan completed before tip-advance window opened; \ - lower the midpoint threshold or use a larger TestChain (got {})", - filter_height, - ); - } - if tokio::time::Instant::now() > inflight_deadline { - panic!( - "rescan committed_height did not reach midpoint {} within 60s, stuck at {}", - midpoint, filter_height, - ); - } - tokio::select! { - _ = client_handle.progress_receiver.changed() => {} - _ = tokio::time::sleep(Duration::from_millis(20)) => {} - } - } - + // Adding W2 with birth_height 0 has dragged committed_height back to + // genesis, so a rescan is now climbing back through the chain. We fund W2 + // and mine immediately: the rescan is still in flight (climbing 40k blocks + // takes real time even from storage), so the tip advance naturally lands + // mid-rescan. We deliberately do NOT try to observe the ephemeral midpoint + // — the progress watch channel coalesces intermediate committed_height + // values, so a storage-only rescan can jump straight to tip and make any + // such observation racy. Only the final converged state is asserted below. let miner_address = ctx.dashd.node.get_new_address_from_wallet("default"); let w2_funding_txid = ctx.dashd.node.send_to_address(&w2_address, Amount::from_sat(60_000_000)); ctx.dashd.node.generate_blocks(5, &miner_address); diff --git a/dash-spv/tests/dashd_sync/tests_transaction.rs b/dash-spv/tests/dashd_sync/tests_transaction.rs index 0dfb5d7aa..3f2f888fd 100644 --- a/dash-spv/tests/dashd_sync/tests_transaction.rs +++ b/dash-spv/tests/dashd_sync/tests_transaction.rs @@ -6,7 +6,7 @@ use std::time::Duration; use tokio::sync::RwLock; use super::helpers::{ - count_wallet_transactions, get_spendable_balance, wait_for_mempool_tx, wait_for_sync, + count_wallet_transactions, get_spendable_balance, wait_for_expected_mempool_tx, wait_for_sync, wait_for_wallet_synced, EMPTY_MNEMONIC, SECONDARY_MNEMONIC, }; use super::setup::{create_and_start_client, TestContext}; @@ -407,9 +407,18 @@ async fn test_spend_change_balance() { build_and_sign(&wallet, &wallet_id, &dest_a, 100_000_000).await.expect("build tx_a"); client_handle.client.broadcast_transaction(&tx_a).await.expect("broadcast tx_a"); - wait_for_mempool_tx(&mut client_handle.wallet_event_receiver, MEMPOOL_TIMEOUT) - .await - .expect("detect tx_a"); + // Match tx_a's exact txid: the funding tx also passes through the mempool, + // and building tx_b below relies on tx_a having been processed into the + // wallet, so a stale funding event must not stand in for tx_a here. + assert!( + wait_for_expected_mempool_tx( + &mut client_handle.wallet_event_receiver, + tx_a.txid(), + MEMPOOL_TIMEOUT, + ) + .await, + "tx_a should be detected in the mempool", + ); // The wallet's only UTXO now is the mempool change from tx_a, so a // successful build proves coin selection used it. @@ -423,9 +432,15 @@ async fn test_spend_change_balance() { ); client_handle.client.broadcast_transaction(&tx_b).await.expect("broadcast tx_b"); - wait_for_mempool_tx(&mut client_handle.wallet_event_receiver, MEMPOOL_TIMEOUT) - .await - .expect("detect tx_b"); + assert!( + wait_for_expected_mempool_tx( + &mut client_handle.wallet_event_receiver, + tx_b.txid(), + MEMPOOL_TIMEOUT, + ) + .await, + "tx_b should be detected in the mempool", + ); client_handle.stop().await; } @@ -481,17 +496,29 @@ async fn test_concurrent_builds_do_not_double_spend() { assert!(!overlap, "tx_a and tx_b must not share an input (double-spend)"); // Both broadcasts succeed and both transactions reach the mempool: a - // double-spend would have the second rejected by the network. + // double-spend would have the second rejected by the network. Match on the + // exact txid — the two funding txs also pass through the mempool, so a stale + // funding event must not be mistaken for the transaction we just broadcast. client_handle.client.broadcast_transaction(&tx_a).await.expect("broadcast tx_a"); - let detected_a = wait_for_mempool_tx(&mut client_handle.wallet_event_receiver, MEMPOOL_TIMEOUT) - .await - .expect("detect tx_a"); - assert_eq!(detected_a, tx_a.txid()); + assert!( + wait_for_expected_mempool_tx( + &mut client_handle.wallet_event_receiver, + tx_a.txid(), + MEMPOOL_TIMEOUT, + ) + .await, + "tx_a should be detected in the mempool", + ); client_handle.client.broadcast_transaction(&tx_b).await.expect("broadcast tx_b"); - let detected_b = wait_for_mempool_tx(&mut client_handle.wallet_event_receiver, MEMPOOL_TIMEOUT) - .await - .expect("detect tx_b"); - assert_eq!(detected_b, tx_b.txid()); + assert!( + wait_for_expected_mempool_tx( + &mut client_handle.wallet_event_receiver, + tx_b.txid(), + MEMPOOL_TIMEOUT, + ) + .await, + "tx_b should be detected in the mempool", + ); client_handle.stop().await; } @@ -516,9 +543,15 @@ async fn test_spend_incoming_balance() { let incoming_amount = Amount::from_sat(300_000_000); let incoming_txid = ctx.dashd.node.send_to_address(&receive_address, incoming_amount); - wait_for_mempool_tx(&mut client_handle.wallet_event_receiver, MEMPOOL_TIMEOUT) - .await - .expect("detect incoming"); + assert!( + wait_for_expected_mempool_tx( + &mut client_handle.wallet_event_receiver, + incoming_txid, + MEMPOOL_TIMEOUT, + ) + .await, + "incoming funding tx should be detected in the mempool", + ); let dest = Address::dummy(Network::Regtest, 3); let (tx, _) = build_and_sign(&wallet, &wallet_id, &dest, 150_000_000) @@ -530,9 +563,15 @@ async fn test_spend_incoming_balance() { ); client_handle.client.broadcast_transaction(&tx).await.expect("broadcast spend"); - wait_for_mempool_tx(&mut client_handle.wallet_event_receiver, MEMPOOL_TIMEOUT) - .await - .expect("detect spend"); + assert!( + wait_for_expected_mempool_tx( + &mut client_handle.wallet_event_receiver, + tx.txid(), + MEMPOOL_TIMEOUT, + ) + .await, + "spend tx should be detected in the mempool", + ); client_handle.stop().await; } @@ -616,9 +655,15 @@ async fn test_drain_account_into_another() { }; client_handle.client.broadcast_transaction(&tx).await.expect("broadcast"); - wait_for_mempool_tx(&mut client_handle.wallet_event_receiver, MEMPOOL_TIMEOUT) - .await - .expect("mempool"); + assert!( + wait_for_expected_mempool_tx( + &mut client_handle.wallet_event_receiver, + tx.txid(), + MEMPOOL_TIMEOUT, + ) + .await, + "drain tx should be detected in the mempool", + ); ctx.dashd.node.generate_blocks(1, &miner); let drained_height = funded_height + 1; diff --git a/dash-spv/tests/peer_test.rs b/dash-spv/tests/peer_test.rs deleted file mode 100644 index 8634d4413..000000000 --- a/dash-spv/tests/peer_test.rs +++ /dev/null @@ -1,232 +0,0 @@ -//! Integration tests for peer networking - -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::Duration; -use tempfile::TempDir; -use tokio::sync::RwLock; -use tokio::time; - -use dash_spv::client::{ClientConfig, DashSpvClient}; -use dash_spv::network::PeerNetworkManager; -use dash_spv::storage::DiskStorageManager; -use dash_spv::types::ValidationMode; -use dashcore::Network; -use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; -use key_wallet_manager::WalletManager; - -fn init_test_tracing() { - let _ = tracing_subscriber::fmt().with_test_writer().try_init(); -} - -/// Create a test configuration with the given network -fn create_test_config(network: Network) -> ClientConfig { - let mut config = ClientConfig::new(network); - - config.storage_path = TempDir::new().unwrap().path().to_path_buf(); - - config.validation_mode = ValidationMode::Basic; - config.enable_filters = false; - config.enable_masternodes = false; - config.max_peers = 3; - config.peers = vec![]; // Will be populated by DNS discovery - config -} - -#[tokio::test] -#[ignore] // Requires network access -async fn test_peer_connection() { - init_test_tracing(); - - let config = create_test_config(Network::Testnet); - - // Create network manager - let network_manager = PeerNetworkManager::new(&config).await.unwrap(); - - // Create storage manager - let storage_manager = DiskStorageManager::new(&config).await.unwrap(); - - // Create wallet manager - let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); - - let client = - DashSpvClient::new(config, network_manager, storage_manager, wallet, vec![]).await.unwrap(); - - let run_client = client.clone(); - let handle = tokio::spawn(async move { run_client.run().await }); - - // Give it time to connect to peers - time::sleep(Duration::from_secs(5)).await; - - // Check that we have connected to at least one peer - let peer_count = client.peer_count().await; - assert!(peer_count > 0, "Should have connected to at least one peer"); - - client.stop().await.expect("Should stop"); - let _ = handle.await; -} - -#[tokio::test] -#[ignore] // Requires network access -async fn test_peer_persistence() { - init_test_tracing(); - - let config = create_test_config(Network::Testnet); - - // First run: connect and save peers - { - // Create network manager - let network_manager = PeerNetworkManager::new(&config).await.unwrap(); - - // Create storage manager - let storage_manager = DiskStorageManager::new(&config).await.unwrap(); - - // Create wallet manager - let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); - - let client = - DashSpvClient::new(config.clone(), network_manager, storage_manager, wallet, vec![]) - .await - .unwrap(); - - let run_client = client.clone(); - let handle = tokio::spawn(async move { run_client.run().await }); - - time::sleep(Duration::from_secs(5)).await; - - let peer_count = client.peer_count().await; - assert!(peer_count > 0, "Should have connected to peers"); - - client.stop().await.expect("Should stop"); - let _ = handle.await; - } - - // Second run: should load saved peers - { - // Create network manager - let network_manager = PeerNetworkManager::new(&config).await.unwrap(); - - // Create storage manager - reuse same path - let storage_manager = DiskStorageManager::new(&config).await.unwrap(); - - // Create wallet manager - let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); - - let client = DashSpvClient::new(config, network_manager, storage_manager, wallet, vec![]) - .await - .unwrap(); - - // Should connect faster due to saved peers - let run_client = client.clone(); - let start = tokio::time::Instant::now(); - let handle = tokio::spawn(async move { run_client.run().await }); - - // Wait for connection but with shorter timeout - time::sleep(Duration::from_secs(3)).await; - - let peer_count = client.peer_count().await; - assert!(peer_count > 0, "Should have connected using saved peers"); - - let elapsed = start.elapsed(); - println!("Connected to {} peers in {:?} (using saved peers)", peer_count, elapsed); - - client.stop().await.expect("Should stop"); - let _ = handle.await; - } -} - -#[tokio::test] -async fn test_peer_disconnection() { - init_test_tracing(); - - let mut config = create_test_config(Network::Regtest); - - // Add manual test peers (would need actual regtest nodes running) - config.peers = vec!["127.0.0.1:19899".parse().unwrap(), "127.0.0.1:19898".parse().unwrap()]; - - // Create network manager - let network_manager = PeerNetworkManager::new(&config).await.unwrap(); - - // Create storage manager - let storage_manager = DiskStorageManager::new(&config).await.unwrap(); - - // Create wallet manager - let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); - - let client = - DashSpvClient::new(config, network_manager, storage_manager, wallet, vec![]).await.unwrap(); - - // Note: This test would require actual regtest nodes running - // For now, we just test that the API works - let test_addr: SocketAddr = "127.0.0.1:19899".parse().unwrap(); - - // Try to disconnect (will fail if not connected, but tests the API) - match client.disconnect_peer(&test_addr, "Test disconnection").await { - Ok(_) => println!("Disconnected peer {}", test_addr), - Err(e) => println!("Expected error disconnecting non-existent peer: {}", e), - } -} - -#[cfg(test)] -mod unit_tests { - use super::*; - use dash_spv::network::addrv2::AddrV2Handler; - use dash_spv::network::discovery::DnsDiscovery; - use dash_spv::network::pool::PeerPool; - use dashcore::network::constants::ServiceFlags; - - #[tokio::test] - async fn test_connection_pool_limits() { - let pool = PeerPool::new(8); - - // Should start empty - assert_eq!(pool.peer_count().await, 0); - assert!(pool.needs_more_peers().await); - assert!(pool.can_accept_peers().await); - - // Test marking as connecting - let addr1: SocketAddr = "127.0.0.1:9999".parse().unwrap(); - assert!(pool.mark_connecting(addr1).await); - assert!(!pool.mark_connecting(addr1).await); // Already marked - assert!(pool.is_connecting(&addr1).await); - } - - #[tokio::test] - async fn test_addrv2_handler() { - let handler = AddrV2Handler::new(); - - // Test tracking AddrV2 support - let peer: SocketAddr = "192.168.1.1:9999".parse().unwrap(); - handler.handle_sendaddrv2(peer).await; - assert!(handler.peer_supports_addrv2(&peer).await); - - // Test adding addresses - handler.add_known_address(peer, ServiceFlags::NETWORK).await; - let known = handler.get_known_addresses().await; - assert_eq!(known.len(), 1); - assert_eq!(known[0].socket_addr().unwrap(), peer); - - // Test getting addresses for sharing - let to_share = handler.get_addresses_for_peer(10).await; - assert_eq!(to_share.len(), 1); - } - - #[tokio::test] - #[ignore] // Requires network access - async fn test_dns_discovery() { - let discovery = DnsDiscovery::new(); - - // Test mainnet discovery - let peers = discovery.discover_peers(Network::Mainnet).await; - assert!(!peers.is_empty(), "Should discover mainnet peers"); - - // All peers should use correct port - for peer in &peers { - assert_eq!(peer.port(), 9999); - } - - // Test limited discovery - let limited = discovery.discover_peers_limited(Network::Mainnet, 5).await; - assert!(limited.len() <= 5); - } -} diff --git a/dash-spv/tests/test_handshake_logic.rs b/dash-spv/tests/test_handshake_logic.rs deleted file mode 100644 index d8ebdb6c9..000000000 --- a/dash-spv/tests/test_handshake_logic.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Unit tests for handshake logic - -use dash_spv::network::{HandshakeManager, HandshakeState}; -use dashcore::Network; - -#[test] -fn test_handshake_state_transitions() { - let mut handshake = HandshakeManager::new(Network::Mainnet, None); - - // Initial state should be Init - assert_eq!(*handshake.state(), HandshakeState::Init); - - // After reset, should be back to Init - handshake.reset(); - assert_eq!(*handshake.state(), HandshakeState::Init); -} diff --git a/dash-spv/tests/wallet_integration_test.rs b/dash-spv/tests/wallet_integration_test.rs index ef374ebfc..dc7afbd72 100644 --- a/dash-spv/tests/wallet_integration_test.rs +++ b/dash-spv/tests/wallet_integration_test.rs @@ -7,15 +7,14 @@ use std::time::Duration; use tempfile::TempDir; use tokio::sync::RwLock; -use dash_spv::network::PeerNetworkManager; use dash_spv::storage::DiskStorageManager; use dash_spv::{ClientConfig, DashSpvClient}; use dashcore::Network; use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; use key_wallet_manager::WalletManager; /// Create a test SPV client with memory storage for integration testing. -async fn create_test_client( -) -> DashSpvClient, PeerNetworkManager, DiskStorageManager> { +async fn create_test_client() -> DashSpvClient, DiskStorageManager> +{ let config = ClientConfig::testnet() .without_filters() .with_storage_path(TempDir::new().unwrap().path()) @@ -23,16 +22,13 @@ async fn create_test_client( // Ensure DNS discovery isn't used since it's causing flakiness in CI and not needed for these tests. .with_restrict_to_configured_peers(true); - // Create network manager - let network_manager = PeerNetworkManager::new(&config).await.unwrap(); - // Create storage manager let storage_manager = DiskStorageManager::new(&config).await.expect("Failed to create storage"); // Create wallet manager let wallet = Arc::new(RwLock::new(WalletManager::::new(config.network))); - DashSpvClient::new(config, network_manager, storage_manager, wallet, vec![]).await.unwrap() + DashSpvClient::new(config, storage_manager, wallet, vec![]).await.unwrap() } #[tokio::test] diff --git a/dash/src/network/message.rs b/dash/src/network/message.rs index d64aaad53..97f66b6d2 100644 --- a/dash/src/network/message.rs +++ b/dash/src/network/message.rs @@ -176,7 +176,6 @@ pub struct RawNetworkMessage { /// A Network message payload. Proper documentation is available on at /// [Bitcoin Wiki: Protocol Specification](https://en.bitcoin.it/wiki/Protocol_specification) #[derive(Clone, PartialEq, Eq, Debug)] -#[allow(clippy::large_enum_variant)] pub enum NetworkMessage { /// `version` Version(message_network::VersionMessage), @@ -197,7 +196,8 @@ pub enum NetworkMessage { /// `mempool` MemPool, /// tx - Tx(transaction::Transaction), + /// Boxed to keep the enum narrow — see the note on `QRInfo`. + Tx(Box), /// `block` Block(block::Block), /// `headers` @@ -262,15 +262,21 @@ pub enum NetworkMessage { /// `getmnlistd` GetMnListD(message_sml::GetMnListDiff), /// `mnlistdiff` - MnListDiff(message_sml::MnListDiff), + /// Boxed to keep the enum narrow — see the note on `QRInfo`. + MnListDiff(Box), /// `getqrinfo` GetQRInfo(message_qrinfo::GetQRInfo), /// `qrinfo` - QRInfo(message_qrinfo::QRInfo), + /// + /// Boxed: a `QRInfo` is 4,336 bytes and every other variant fits in under 700, so + /// inlining it would make *every* `NetworkMessage` — each `ping`, `inv` and `block` — + /// 4 KB wide, in every queue that holds one. + QRInfo(Box), /// `clsig` CLSig(ChainLock), /// `isdlock` - ISLock(InstantLock), + /// Boxed to keep the enum narrow — see the note on `QRInfo`. + ISLock(Box), /// `senddsq` - Notify peer whether to send CoinJoin queue messages SendDsq(bool), /// Any other message. @@ -398,7 +404,7 @@ impl Encodable for RawNetworkMessage { NetworkMessage::NotFound(ref dat) => serialize(dat), NetworkMessage::GetBlocks(ref dat) => serialize(dat), NetworkMessage::GetHeaders(ref dat) => serialize(dat), - NetworkMessage::Tx(ref dat) => serialize(dat), + NetworkMessage::Tx(ref dat) => serialize(dat.as_ref()), NetworkMessage::Block(ref dat) => serialize(dat), NetworkMessage::Headers(ref dat) => serialize(&HeaderSerializationWrapper(dat)), NetworkMessage::GetHeaders2(ref dat) => serialize(dat), @@ -435,11 +441,11 @@ impl Encodable for RawNetworkMessage { .. } => serialize(data), NetworkMessage::GetMnListD(ref dat) => serialize(dat), - NetworkMessage::MnListDiff(ref dat) => serialize(dat), + NetworkMessage::MnListDiff(ref dat) => serialize(dat.as_ref()), NetworkMessage::GetQRInfo(ref dat) => serialize(dat), - NetworkMessage::QRInfo(ref dat) => serialize(dat), + NetworkMessage::QRInfo(ref dat) => serialize(dat.as_ref()), NetworkMessage::CLSig(ref dat) => serialize(dat), - NetworkMessage::ISLock(ref dat) => serialize(dat), + NetworkMessage::ISLock(ref dat) => serialize(dat.as_ref()), NetworkMessage::SendDsq(wants_dsq) => serialize(&(wants_dsq as u8)), }) .consensus_encode(w)?; @@ -582,7 +588,9 @@ impl Decodable for RawNetworkMessage { Decodable::consensus_decode_from_finite_reader(&mut mem_d)?, ), "filterclear" => NetworkMessage::FilterClear, - "tx" => NetworkMessage::Tx(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?), + "tx" => NetworkMessage::Tx(Box::new(Decodable::consensus_decode_from_finite_reader( + &mut mem_d, + )?)), "getcfilters" => NetworkMessage::GetCFilters( Decodable::consensus_decode_from_finite_reader(&mut mem_d)?, ), @@ -630,21 +638,21 @@ impl Decodable for RawNetworkMessage { "getmnlistd" => NetworkMessage::GetMnListD( Decodable::consensus_decode_from_finite_reader(&mut mem_d)?, ), - "mnlistdiff" => NetworkMessage::MnListDiff( + "mnlistdiff" => NetworkMessage::MnListDiff(Box::new( Decodable::consensus_decode_from_finite_reader(&mut mem_d)?, - ), + )), "getqrinfo" => NetworkMessage::GetQRInfo( Decodable::consensus_decode_from_finite_reader(&mut mem_d)?, ), - "qrinfo" => { - NetworkMessage::QRInfo(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?) - } + "qrinfo" => NetworkMessage::QRInfo(Box::new( + Decodable::consensus_decode_from_finite_reader(&mut mem_d)?, + )), "clsig" => { NetworkMessage::CLSig(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?) } - "isdlock" => { - NetworkMessage::ISLock(Decodable::consensus_decode_from_finite_reader(&mut mem_d)?) - } + "isdlock" => NetworkMessage::ISLock(Box::new( + Decodable::consensus_decode_from_finite_reader(&mut mem_d)?, + )), "senddsq" => { let byte: u8 = Decodable::consensus_decode_from_finite_reader(&mut mem_d)?; NetworkMessage::SendDsq(byte != 0) @@ -871,7 +879,7 @@ mod test { hash_x11([50u8; 32]).into(), )), NetworkMessage::MemPool, - NetworkMessage::Tx(tx), + NetworkMessage::Tx(Box::new(tx)), NetworkMessage::Block(block), NetworkMessage::Headers(vec![header]), NetworkMessage::SendHeaders, diff --git a/dash/src/network/message_sml.rs b/dash/src/network/message_sml.rs index 9212811d8..19526881b 100644 --- a/dash/src/network/message_sml.rs +++ b/dash/src/network/message_sml.rs @@ -119,10 +119,10 @@ mod tests { let data = hex::decode(block_hex).expect("decode hex"); let mn_list_diff: RawNetworkMessage = deserialize(&data).expect("deserialize MnListDiff"); if let NetworkMessage::MnListDiff(diff) = mn_list_diff.payload { - let serialized = serialize(&diff); + let serialized = serialize(diff.as_ref()); let deserialized: MnListDiff = deserialize(serialized.as_slice()).expect("expected to deserialize"); - assert_eq!(deserialized, diff); + assert_eq!(deserialized, *diff); } } } diff --git a/masternode-seeds-fetcher/Cargo.toml b/masternode-seeds-fetcher/Cargo.toml index 012dcb74a..981f94431 100644 --- a/masternode-seeds-fetcher/Cargo.toml +++ b/masternode-seeds-fetcher/Cargo.toml @@ -13,11 +13,12 @@ name = "masternode-seeds-fetcher" path = "src/main.rs" [dependencies] -dashcore = { path = "../dash", features = ["core-block-hash-use-x11"] } +dashcore = { path = "../dash", features = ["core-block-hash-use-x11", "tokio"] } dash-spv = { path = "../dash-spv" } dash-network-seeds = { path = "../dash-network-seeds" } tokio = { version = "1.0", features = ["full"] } +tokio-util = "0.7" clap = { version = "4.0", features = ["derive", "env"] } anyhow = "1.0" diff --git a/masternode-seeds-fetcher/src/main.rs b/masternode-seeds-fetcher/src/main.rs index 6b1f68613..4fba6245a 100644 --- a/masternode-seeds-fetcher/src/main.rs +++ b/masternode-seeds-fetcher/src/main.rs @@ -33,12 +33,12 @@ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::time::Duration; +use crate::peer::Peer; use anyhow::{Context, Result, anyhow}; use clap::Parser; use dash_network_seeds::{ CoreStatus, MasternodeSeed, MasternodeType, PlatformStatus, Reachability, }; -use dash_spv::network::Peer; use dashcore::hashes::Hash; use dashcore::network::Address; use dashcore::network::constants::ServiceFlags; @@ -53,6 +53,7 @@ use std::sync::Arc; use tokio::sync::Semaphore; use tokio::time::Instant; +mod peer; mod probe; // ---------- CLI ---------- @@ -256,7 +257,7 @@ async fn fetch_from_peer(peer_addr: SocketAddr, network: Network) -> Result Some(d.clone()), + NetworkMessage::MnListDiff(d) => Some((**d).clone()), _ => None, }) .await diff --git a/masternode-seeds-fetcher/src/peer.rs b/masternode-seeds-fetcher/src/peer.rs new file mode 100644 index 000000000..5d6ee16f6 --- /dev/null +++ b/masternode-seeds-fetcher/src/peer.rs @@ -0,0 +1,74 @@ +//! Minimal Dash P2P connection for probing a single peer. +//! +//! The SPV client's network module drives its peers from background tasks and hands +//! messages to subscribers — the right shape for a sync, the wrong one for a probe that +//! wants to send a request and block on the reply. So this tool keeps its own socket +//! rather than depending on the client's internals. + +use std::net::SocketAddr; +use std::time::Duration; + +use anyhow::{Context, Result, anyhow}; +use dashcore::Network; +use dashcore::consensus::encode; +use dashcore::network::message::{NetworkMessage, RawNetworkMessage, RawNetworkMessageCodec}; +use futures::StreamExt; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpStream; +use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; +use tokio_util::codec::FramedRead; + +/// A message received from a peer. +pub struct Message(NetworkMessage); + +impl Message { + pub fn inner(&self) -> &NetworkMessage { + &self.0 + } +} + +/// A connected peer: a framed reader and a raw writer over one TCP socket. +pub struct Peer { + reader: FramedRead, + writer: OwnedWriteHalf, + magic: u32, +} + +impl Peer { + /// Open a TCP connection to `addr`. No handshake — the caller drives it. + pub async fn connect(addr: SocketAddr, timeout_secs: u64, network: Network) -> Result { + let stream = + tokio::time::timeout(Duration::from_secs(timeout_secs), TcpStream::connect(addr)) + .await + .map_err(|_| anyhow!("connect to {addr} timed out after {timeout_secs}s"))? + .with_context(|| format!("connect to {addr}"))?; + + let (read_half, writer) = stream.into_split(); + + Ok(Self { + reader: FramedRead::new(read_half, RawNetworkMessageCodec), + writer, + magic: network.magic(), + }) + } + + pub async fn send_message(&mut self, message: NetworkMessage) -> Result<()> { + let raw = RawNetworkMessage { + magic: self.magic, + payload: message, + }; + + self.writer.write_all(&encode::serialize(&raw)).await.context("send message")?; + + Ok(()) + } + + /// Next message from the peer, or `None` once it closes the connection. + pub async fn receive_message(&mut self) -> Result> { + match self.reader.next().await { + Some(Ok(raw)) => Ok(Some(Message(raw.payload))), + Some(Err(e)) => Err(e).context("decode message"), + None => Ok(None), + } + } +} diff --git a/masternode-seeds-fetcher/src/probe.rs b/masternode-seeds-fetcher/src/probe.rs index 49a1de9cc..3020cbe45 100644 --- a/masternode-seeds-fetcher/src/probe.rs +++ b/masternode-seeds-fetcher/src/probe.rs @@ -27,7 +27,7 @@ use tokio::net::TcpStream; use x509_parser::prelude::FromDer; use x509_parser::x509::X509Version; -use dash_spv::network::Peer; +use crate::peer::Peer; /// How long to give a single TCP connect before giving up. const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); From fed5b64d8e1edaf509b6599a28f377cccfadfa8c Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 09:48:04 +0000 Subject: [PATCH 04/25] dash-spv-bench extra features --- dash-spv-bench/Cargo.toml | 1 + dash-spv-bench/scenarios/real-mainnet.yml | 8 +- dash-spv-bench/scenarios/real-testnet.yml | 2 +- dash-spv-bench/src/dashboard.rs | 269 ++++++++++++++++++++++ dash-spv-bench/src/main.rs | 48 +++- dash-spv-bench/src/metrics.rs | 24 +- 6 files changed, 334 insertions(+), 18 deletions(-) create mode 100644 dash-spv-bench/src/dashboard.rs diff --git a/dash-spv-bench/Cargo.toml b/dash-spv-bench/Cargo.toml index c0201922e..776fa96d3 100644 --- a/dash-spv-bench/Cargo.toml +++ b/dash-spv-bench/Cargo.toml @@ -18,6 +18,7 @@ tempfile = "3.0" tracing = "0.1" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } +indicatif = "0.17" [[bin]] name = "dash-spv-bench" diff --git a/dash-spv-bench/scenarios/real-mainnet.yml b/dash-spv-bench/scenarios/real-mainnet.yml index 1d053541e..1bbffe7f6 100644 --- a/dash-spv-bench/scenarios/real-mainnet.yml +++ b/dash-spv-bench/scenarios/real-mainnet.yml @@ -1,4 +1,10 @@ -description: "Mainnet sync from a 2M checkpoint via DNS peer discovery." +description: "Mainnet sync from genesis via DNS peer discovery." cpus: "0-11" max_peers: 3 mode: mainnet +# NOTE: `start_height: 2000000` (what the description used to claim) does NOT work +# yet: verifying the first filter batch needs the filter header at start-1, which a +# checkpoint sync never downloads ("Missing filter header at height 1999999"). Until +# the checkpoint carries a trusted filter header, this syncs from genesis — which +# means ~2.5M cfilters (~1.8GB), so the phase is bandwidth-bound and its time is +# mostly bytes on the wire, not client work. diff --git a/dash-spv-bench/scenarios/real-testnet.yml b/dash-spv-bench/scenarios/real-testnet.yml index b3435857f..c942f0217 100644 --- a/dash-spv-bench/scenarios/real-testnet.yml +++ b/dash-spv-bench/scenarios/real-testnet.yml @@ -1,4 +1,4 @@ description: "Testnet sync from a 500k checkpoint via DNS peer discovery." cpus: "0-11" -max_peers: 3 +max_peers: 16 mode: testnet diff --git a/dash-spv-bench/src/dashboard.rs b/dash-spv-bench/src/dashboard.rs new file mode 100644 index 000000000..7b15611fa --- /dev/null +++ b/dash-spv-bench/src/dashboard.rs @@ -0,0 +1,269 @@ +//! Live progress bars pinned to the bottom of the terminal, cargo-style. +//! +//! Fed by `EventHandler::on_progress`, which the client calls on every progress +//! change. All four bars exist from the start and stay together at the bottom: +//! `indicatif` owns those lines and redraws them in place, so they never scroll +//! away as the sync advances. +//! +//! Log lines must go through [`Dashboard::log_writer`] (wired into the tracing +//! subscriber in `main`). Writing to stderr directly would print INTO the lines +//! the bars own and tear them apart; the writer suspends the bars, prints, and +//! lets them redraw underneath. + +use std::io::{self, IsTerminal, Write}; + +use dash_spv::sync::{ProgressPercentage, SyncProgress}; +use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; +use tracing_subscriber::fmt::MakeWriter; + +/// Repaint ceiling. Progress callbacks arrive far faster than a terminal (or an +/// eye) can follow, and each repaint is a write syscall. +const REDRAW_HZ: u8 = 10; + +const BAR_WIDTH: usize = 32; + +/// Placeholder for a manager the client never started (the scenario disabled it). +/// Doubles as the marker that a bar holds no real data — see `finish`. +const NOT_ENABLED: &str = "not enabled"; + +pub struct Dashboard { + multi: MultiProgress, + // Height-ranged phases: a bar that fills toward the chain tip. + block_headers: ProgressBar, + filter_headers: ProgressBar, + filters: ProgressBar, + masternodes: ProgressBar, + // Count-based phases: a bar over "how many of what we asked for". + blocks: ProgressBar, + // Event-driven phases: nothing to fill — they just report what they have seen. + chainlocks: ProgressBar, + instantsend: ProgressBar, + mempool: ProgressBar, +} + +impl Dashboard { + pub fn new() -> Self { + // Hidden when stderr is not a TTY (CI, redirected run): the escapes would + // just be noise in the log file, and `run.sh` parses that output. + let target = if io::stderr().is_terminal() { + ProgressDrawTarget::stderr_with_hz(REDRAW_HZ) + } else { + ProgressDrawTarget::hidden() + }; + let multi = MultiProgress::with_draw_target(target); + + let bar_style = ProgressStyle::with_template(&format!( + " {{prefix:<15}} [{{bar:{BAR_WIDTH}}}] {{percent:>3}}% {{msg}}" + )) + .expect("static template") + .progress_chars("=> "); + + // Chainlocks / InstantSend / mempool never "complete" — they run for as long + // as the client does — so a bar would be a lie. Same column layout, blank + // where the bar would be, so every pipeline still lines up in one table. + let info_style = ProgressStyle::with_template(&format!( + " {{prefix:<15}} {:BAR_WIDTH$} {{msg}}", + "" + )) + .expect("static template"); + + let mk = |label: &'static str, style: &ProgressStyle| { + let pb = multi.add(ProgressBar::new(1)); + pb.set_style(style.clone()); + pb.set_prefix(label); + pb.set_message(NOT_ENABLED); + pb + }; + + Self { + block_headers: mk("block headers", &bar_style), + filter_headers: mk("filter headers", &bar_style), + filters: mk("filters", &bar_style), + masternodes: mk("masternodes", &bar_style), + blocks: mk("matched blocks", &bar_style), + chainlocks: mk("chainlocks", &info_style), + instantsend: mk("instantsend", &info_style), + mempool: mk("mempool", &info_style), + multi, + } + } + + /// Writer for the tracing subscriber: suspends the bars around each log line + /// so the two never overwrite each other. + pub fn log_writer(&self) -> BarWriter { + BarWriter(self.multi.clone()) + } + + /// Push the latest progress into the bars. + pub fn render(&self, progress: &SyncProgress) { + /// Drive a bar from the phase's OWN progress reporting — the same + /// `current/target` and percentage its `Display` prints in the info logs — + /// so the bar and the log line can never disagree. + fn set(pb: &ProgressBar, p: &impl ProgressPercentage) { + let (current, target) = (p.current_height(), p.target_height()); + // A zero target means the phase has not learned its range yet: leave the + // bar empty rather than reading a bogus 100%. + pb.set_length(target.max(1) as u64); + pb.set_position(current.min(target) as u64); + pb.set_message(format!("{} / {}", current, target)); + } + + if let Ok(p) = progress.headers() { + set(&self.block_headers, p); + } + if let Ok(p) = progress.filter_headers() { + // Download progress, not the verified tip: cfheaders come down in + // parallel and verify strictly in order, so the verified prefix sits + // still and then jumps. This is the number the log line shows. + let (current, target) = (p.effective_height(), p.target_height()); + self.filter_headers.set_length(target.max(1) as u64); + self.filter_headers.set_position(current.min(target) as u64); + self.filter_headers.set_message(format!( + "{} / {} (verified {})", + current, + target, + p.current_height() + )); + } + if let Ok(p) = progress.filters() { + set(&self.filters, p); + } + if let Ok(p) = progress.masternodes() { + // Masternodes has the same current/target pair (its log prints exactly + // these) but does not implement the trait, so drive it by hand. + let (current, target) = (p.current_height(), p.target_height()); + self.masternodes.set_length(target.max(1) as u64); + self.masternodes.set_position(current.min(target) as u64); + self.masternodes.set_message(format!( + "{} / {} ({} diffs, {} cycles)", + current, + target, + p.diffs_processed(), + p.validated_cycles() + )); + } + if let Ok(p) = progress.blocks() { + // Track how far along the CHAIN the matched blocks have been processed, + // not processed-of-requested: the request count keeps growing as the + // scan finds more matches, so a bar against it walks backwards. Height + // against the tip only ever moves forward. + let tip = progress.headers().map(|h| h.target_height()).unwrap_or(0); + let done = p.last_processed(); + self.blocks.set_length(tip.max(1) as u64); + self.blocks.set_position(done.min(tip) as u64); + self.blocks.set_message(format!( + "{} / {} ({} blocks, {} txs)", + done, + tip, + p.processed(), + p.transactions() + )); + } + if let Ok(p) = progress.chainlocks() { + self.chainlocks.set_message(format!( + "{:?} best {} ({} valid, {} invalid)", + p.state(), + p.best_validated_height(), + p.valid(), + p.invalid() + )); + } + if let Ok(p) = progress.instantsend() { + self.instantsend.set_message(format!( + "{:?} {} valid, {} invalid, {} pending", + p.state(), + p.valid(), + p.invalid(), + p.pending() + )); + } + if let Ok(p) = progress.mempool() { + self.mempool.set_message(format!( + "{:?} {} tracked ({} relevant of {} seen)", + p.state(), + p.tracked(), + p.relevant(), + p.received() + )); + } + } + + /// Freeze the bars where they are, so the results summary prints below them. + pub fn finish(&self) { + for pb in [ + &self.block_headers, + &self.filter_headers, + &self.filters, + &self.masternodes, + &self.blocks, + &self.chainlocks, + &self.instantsend, + &self.mempool, + ] { + // `finish` fills a bar to its length — right for a phase that ran, a lie + // for one that never started (a disabled manager would end up reading + // 100%). Those keep their placeholder message, so leave them where they + // are. + if pb.message() == NOT_ENABLED { + pb.abandon(); + } else { + pb.finish(); + } + } + } +} + +impl Default for Dashboard { + fn default() -> Self { + Self::new() + } +} + +/// `MakeWriter` that prints through `MultiProgress::suspend`, so a log line lifts +/// the bars, prints above them, and lets them redraw at the bottom. +#[derive(Clone)] +pub struct BarWriter(MultiProgress); + +impl<'a> MakeWriter<'a> for BarWriter { + type Writer = BarWriterGuard; + + fn make_writer(&'a self) -> Self::Writer { + BarWriterGuard { + multi: self.0.clone(), + buf: Vec::new(), + } + } +} + +/// Buffers one log record and emits it (suspending the bars) when the subscriber +/// flushes / drops it, so a record is never split across a bar redraw. +pub struct BarWriterGuard { + multi: MultiProgress, + buf: Vec, +} + +impl Write for BarWriterGuard { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.buf.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + if self.buf.is_empty() { + return Ok(()); + } + let line = std::mem::take(&mut self.buf); + self.multi.suspend(|| { + let mut err = io::stderr().lock(); + let _ = err.write_all(&line); + let _ = err.flush(); + }); + Ok(()) + } +} + +impl Drop for BarWriterGuard { + fn drop(&mut self) { + let _ = self.flush(); + } +} diff --git a/dash-spv-bench/src/main.rs b/dash-spv-bench/src/main.rs index dd2e2de6b..d3c6c73d6 100644 --- a/dash-spv-bench/src/main.rs +++ b/dash-spv-bench/src/main.rs @@ -1,3 +1,4 @@ +mod dashboard; mod metrics; use std::net::SocketAddr; @@ -5,7 +6,6 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, Context, Result}; -use dash_spv::network::PeerNetworkManager; use dash_spv::storage::DiskStorageManager; use dash_spv::{ClientConfig, DashSpvClient, EventHandler}; use dashcore::Network; @@ -15,6 +15,7 @@ use key_wallet_manager::WalletManager; use std::collections::BTreeSet; use tokio::sync::RwLock; +use crate::dashboard::Dashboard; use crate::metrics::BenchEventHandler; /// Default wallet: a mnemonic with real testnet history @@ -30,12 +31,17 @@ async fn main() -> Result<()> { // Anchor the timeline at process start so `tlog!` timestamps read as time-since-launch. dash_spv::timer::init(); + // The progress bars own the bottom lines of the terminal, so the logs have to + // go through them (they suspend the bars, print, and let them redraw) — writing + // to stderr directly would tear the bars apart. + let dashboard = Arc::new(Dashboard::new()); + tracing_subscriber::fmt() // Logs go to stderr so they stream live and stay separate from the // machine-readable result summary on stdout (which run.sh captures/parses). // Set RUST_LOG to raise verbosity, e.g. // RUST_LOG="warn,dash_spv::network=info" ./run.sh scenarios/.yml - .with_writer(std::io::stderr) + .with_writer(dashboard.log_writer()) .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env() // Note: `dash_spv=warn` would also silence `dash_spv_bench` (prefix match), so @@ -112,24 +118,28 @@ async fn main() -> Result<()> { config.add_peer(*addr); } - let network_manager = - PeerNetworkManager::new(&config).await.map_err(|e| anyhow!("network manager: {e}"))?; let storage_manager = DiskStorageManager::new(&config).await.map_err(|e| anyhow!("storage manager: {e}"))?; // Wallet: a single BIP44 account 0 from the mnemonic, so filter matches drive block download. let mut wallet_manager = WalletManager::::new(network); if !mnemonic.is_empty() { + // Birth height must be the checkpoint we sync from, not 0: the client only + // downloads filters at or above `start_from_height`, so a wallet claiming + // to need scanning from 0 is permanently "behind" the filter progress and + // the filters manager's stale-wallet check restarts the scan forever. + let birth_height = start_height.unwrap_or(0); wallet_manager - .create_wallet_from_mnemonic(&mnemonic, 0, account_creation_options()) + .create_wallet_from_mnemonic(&mnemonic, birth_height, account_creation_options()) .map_err(|e| anyhow!("wallet from mnemonic: {e}"))?; } let wallet = Arc::new(RwLock::new(wallet_manager)); + // Keep a handle for the post-sync correctness fingerprint below. + let wallet_probe = wallet.clone(); - let handler = Arc::new(BenchEventHandler::new()); + let handler = Arc::new(BenchEventHandler::new(dashboard.clone())); let client = DashSpvClient::new( config, - network_manager, storage_manager, wallet, vec![handler.clone() as Arc], @@ -143,6 +153,16 @@ async fn main() -> Result<()> { let _ = run_client.run().await; }); + // Ctrl-C: shut the client down cleanly (cancels the network tasks and + // disconnects peers) instead of leaving readers streaming messages. + let stop_client = client.clone(); + tokio::spawn(async move { + if tokio::signal::ctrl_c().await.is_ok() { + eprintln!("ctrl-c received, shutting down..."); + let _ = stop_client.shutdown().await; + } + }); + let _ = tokio::time::timeout(timeout, handler.wait_done()).await; let m = handler.snapshot(); dash_spv::timer::dump_profile(); @@ -153,6 +173,20 @@ async fn main() -> Result<()> { drop(storage_dir); println!("\n{m}"); + + // Wallet correctness fingerprint. Two runs that sync the SAME wallet must + // land on identical numbers no matter how the filter/rescan pipeline is + // ordered — a mismatch means a matched block (and thus derived addresses / + // funds) was missed. Used to A/B the eager-fanout rescan against the serial + // one: `wallet_addresses` is the gap-limit walk output, the tell-tale. + if !mnemonic.is_empty() { + use key_wallet_manager::WalletInterface; + let w = wallet_probe.read().await; + println!("wallet_synced_height: {}", w.synced_height()); + println!("wallet_addresses: {}", w.monitored_addresses().len()); + println!("wallet_describe:\n{}", w.describe().await); + } + Ok(()) } diff --git a/dash-spv-bench/src/metrics.rs b/dash-spv-bench/src/metrics.rs index 328880dad..04f02236a 100644 --- a/dash-spv-bench/src/metrics.rs +++ b/dash-spv-bench/src/metrics.rs @@ -1,12 +1,14 @@ use std::collections::BTreeMap; use std::fmt; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use std::time::Instant; -use dash_spv::sync::SyncEvent; +use dash_spv::sync::{SyncEvent, SyncProgress}; use dash_spv::EventHandler; use tokio::sync::Notify; +use crate::dashboard::Dashboard; + /// Milestones we time, in pipeline order. Stable string keys so timings are easy to correlate. pub const MILESTONE_BLOCK_HEADERS: &str = "block_headers_complete"; pub const MILESTONE_FILTER_HEADERS: &str = "filter_headers_complete"; @@ -61,6 +63,8 @@ pub struct BenchEventHandler { inner: Mutex, /// Notified once the run reaches `sync_complete`. done: Notify, + /// Live progress bars at the bottom of the terminal. + dashboard: Arc, } struct Inner { @@ -71,7 +75,7 @@ struct Inner { } impl BenchEventHandler { - pub fn new() -> Self { + pub fn new(dashboard: Arc) -> Self { Self { start: Instant::now(), inner: Mutex::new(Inner { @@ -80,6 +84,7 @@ impl BenchEventHandler { completed: false, }), done: Notify::new(), + dashboard, } } @@ -121,12 +126,6 @@ impl BenchEventHandler { } } -impl Default for BenchEventHandler { - fn default() -> Self { - Self::new() - } -} - impl EventHandler for BenchEventHandler { fn on_sync_event(&self, event: &SyncEvent) { match event { @@ -143,6 +142,8 @@ impl EventHandler for BenchEventHandler { .. } => { self.record(MILESTONE_SYNC); + // Let the last frame stand, so the summary prints below it. + self.dashboard.finish(); let mut inner = self.inner.lock().unwrap(); inner.completed = true; drop(inner); @@ -152,6 +153,11 @@ impl EventHandler for BenchEventHandler { } } + /// Every progress change repaints the bars (throttled inside). + fn on_progress(&self, progress: &SyncProgress) { + self.dashboard.render(progress); + } + fn on_error(&self, error: &str) { let mut inner = self.inner.lock().unwrap(); if inner.error.is_none() { From f4591f3997b6bf382ceb058c385cf476c95ee55c Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 11:10:21 +0000 Subject: [PATCH 05/25] dash spv storage versioning --- dash-spv/src/storage/mod.rs | 71 ++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/dash-spv/src/storage/mod.rs b/dash-spv/src/storage/mod.rs index 73135493b..2bf08e926 100644 --- a/dash-spv/src/storage/mod.rs +++ b/dash-spv/src/storage/mod.rs @@ -12,7 +12,7 @@ mod masternode; mod metadata; mod peers; mod segments; -use crate::error::StorageResult; +use crate::error::{StorageError, StorageResult}; use crate::storage::lockfile::LockFile; use crate::types::{HashedBlock, HashedBlockHeader}; use crate::ClientConfig; @@ -121,6 +121,73 @@ fn lock_file_path(storage_path: &Path) -> PathBuf { } } +const STORAGE_VERSION: u32 = 1; + +fn metadata_file(storage_path: &Path) -> PathBuf { + storage_path.join("metadata.json") +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct StorageMeta { + version: u32, +} + +fn read_storage_version(storage_path: &Path) -> u32 { + std::fs::read(metadata_file(storage_path)) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .map_or(0, |meta| meta.version) +} + +fn migrate_storage(storage_path: &Path) -> StorageResult<()> { + let mut version = read_storage_version(storage_path); + + if version >= STORAGE_VERSION { + return Ok(()); + } + + while version < STORAGE_VERSION { + match version { + // 0 -> 1: the scan-state, segment and peer formats changed. + // Rather than risk misreading old data, start from a clean + // storage and re-sync + 0 => wipe_storage(storage_path)?, + other => { + return Err(StorageError::Corruption(format!( + "no migration path from storage version {other}" + ))); + } + } + + version += 1; + } + + write_storage_version(storage_path, version) +} + +fn wipe_storage(storage_path: &Path) -> StorageResult<()> { + for entry in std::fs::read_dir(storage_path)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + std::fs::remove_dir_all(entry.path())?; + } else { + std::fs::remove_file(entry.path())?; + } + } + Ok(()) +} + +fn write_storage_version(storage_path: &Path, version: u32) -> StorageResult<()> { + let json = serde_json::to_vec_pretty(&StorageMeta { + version, + }) + .map_err(|e| StorageError::Serialization(e.to_string()))?; + + std::fs::write(metadata_file(storage_path), json)?; + + Ok(()) +} + impl DiskStorageManager { pub async fn new(config: &ClientConfig) -> StorageResult { use std::fs; @@ -132,6 +199,8 @@ impl DiskStorageManager { let lock_file = LockFile::new(lock_file)?; + migrate_storage(&storage_path)?; + let mut storage = Self { storage_path: storage_path.clone(), From ab97c54aced03ad140f699c547625a8e3a8ae19c Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 11:31:58 +0000 Subject: [PATCH 06/25] drop peer storage until a new one adjusted to the changes is written --- dash-spv/src/network/discovery.rs | 40 +---- dash-spv/src/storage/mod.rs | 2 - dash-spv/src/storage/peers.rs | 160 ------------------ dash-spv/tests/dashd_sync/setup.rs | 21 +-- dash-spv/tests/dashd_sync/tests_disconnect.rs | 13 +- 5 files changed, 18 insertions(+), 218 deletions(-) delete mode 100644 dash-spv/src/storage/peers.rs diff --git a/dash-spv/src/network/discovery.rs b/dash-spv/src/network/discovery.rs index aa335053d..7256accd2 100644 --- a/dash-spv/src/network/discovery.rs +++ b/dash-spv/src/network/discovery.rs @@ -1,22 +1,16 @@ use std::net::SocketAddr; -use std::path::{Path, PathBuf}; use dashcore::Network; use rand::seq::SliceRandom; use crate::network::peer::DisconnectedPeer; -use crate::storage::{PeerStorage, PersistentPeerStorage, PersistentStorage}; use crate::ClientConfig; pub struct PeerDiscoverer { network: Network, - // Empty means "discover": the persisted peer store first, then the seeds. + // Empty means "discover" from the compiled-in seeds, then DNS. fixed: Vec, - // When true, hand out only `fixed`: never touch the peer store, seeds, or DNS. - // With no configured peers this yields an empty set, so no outbound connections - // are made — honoring `ClientConfig::restrict_to_configured_peers`. restrict_to_configured_peers: bool, - storage_path: PathBuf, /// Discovered addresses, resolved once and then kept. /// /// Deliberately not consumed as it is handed out: the reconnector comes back here @@ -31,7 +25,6 @@ impl PeerDiscoverer { network: config.network, fixed: config.peers.clone(), restrict_to_configured_peers: config.restrict_to_configured_peers, - storage_path: config.storage_path.clone(), discovered: None, } } @@ -41,12 +34,10 @@ impl PeerDiscoverer { let pool = if !self.fixed.is_empty() { &self.fixed } else if self.restrict_to_configured_peers { - // Restricted to configured peers, of which there are none: hand out - // nothing rather than falling back to the peer store, seeds, or DNS. return Vec::new(); } else { if self.discovered.is_none() { - let found = Self::discover(self.network, &self.storage_path).await; + let found = Self::discover(self.network).await; self.discovered = Some(found); } self.discovered.as_ref().expect("just set") @@ -57,30 +48,9 @@ impl PeerDiscoverer { .collect() } - /// Addresses to try, best source first: peers we have actually talked to before, then - /// the compiled-in seeds, then DNS. - async fn discover(network: Network, storage_path: &Path) -> Vec { - let mut addresses = Vec::new(); - - // Peers persisted by an earlier run. On regtest/devnet this is the ONLY source — - // there are no DNS seeds to fall back to — so a client configured without explicit - // peers depends on it entirely. - match PersistentPeerStorage::open(storage_path.to_path_buf()).await { - Ok(store) => match store.load_peers().await { - Ok(peers) => { - let known: Vec = - peers.iter().filter_map(|p| p.socket_addr().ok()).collect(); - if !known.is_empty() { - tracing::info!("Peer store: {} known address(es)", known.len()); - } - addresses.extend(known); - } - Err(e) => tracing::warn!("Failed to load the peer store: {e}"), - }, - Err(e) => tracing::warn!("Failed to open the peer store: {e}"), - } - - addresses.extend(dash_network_seeds::addresses(network)); + /// Addresses to try: the compiled-in seeds, then DNS + async fn discover(network: Network) -> Vec { + let mut addresses = dash_network_seeds::addresses(network); let port = network.default_p2p_port(); for seed in network.dns_seeds() { diff --git a/dash-spv/src/storage/mod.rs b/dash-spv/src/storage/mod.rs index 2bf08e926..0b3e2926d 100644 --- a/dash-spv/src/storage/mod.rs +++ b/dash-spv/src/storage/mod.rs @@ -10,7 +10,6 @@ mod io; mod lockfile; mod masternode; mod metadata; -mod peers; mod segments; use crate::error::{StorageError, StorageResult}; use crate::storage::lockfile::LockFile; @@ -33,7 +32,6 @@ pub use crate::storage::filter_headers::{FilterHeaderStorage, PersistentFilterHe pub use crate::storage::filters::{FilterStorage, PersistentFilterStorage}; pub use crate::storage::masternode::{MasternodeStateStorage, PersistentMasternodeStateStorage}; pub use crate::storage::metadata::{MetadataStorage, PersistentMetadataStorage}; -pub use crate::storage::peers::{PeerStorage, PersistentPeerStorage}; pub use types::*; diff --git a/dash-spv/src/storage/peers.rs b/dash-spv/src/storage/peers.rs deleted file mode 100644 index cdf818fe1..000000000 --- a/dash-spv/src/storage/peers.rs +++ /dev/null @@ -1,160 +0,0 @@ -use std::{fs::File, io::BufReader, path::PathBuf}; - -use tokio::fs; - -use async_trait::async_trait; -use dashcore::{ - consensus::{encode, Decodable, Encodable}, - network::address::AddrV2Message, -}; - -use crate::{ - error::StorageResult, - storage::{io::atomic_write, PersistentStorage}, - StorageError, -}; - -#[async_trait] -pub trait PeerStorage { - async fn save_peers( - &self, - peers: &[dashcore::network::address::AddrV2Message], - ) -> StorageResult<()>; - - async fn load_peers(&self) -> StorageResult>; -} - -pub struct PersistentPeerStorage { - storage_path: PathBuf, -} - -impl PersistentPeerStorage { - const FOLDER_NAME: &str = "peers"; - - fn peers_data_file(&self) -> PathBuf { - self.storage_path.join("peers.dat") - } -} - -#[async_trait] -impl PersistentStorage for PersistentPeerStorage { - async fn open(storage_path: impl Into + Send) -> StorageResult { - let storage_path = storage_path.into(); - - Ok(PersistentPeerStorage { - storage_path: storage_path.join(Self::FOLDER_NAME), - }) - } - - async fn persist(&mut self, _storage_path: impl Into + Send) -> StorageResult<()> { - // Current implementation persists data everytime data is stored - Ok(()) - } -} - -#[async_trait] -impl PeerStorage for PersistentPeerStorage { - async fn save_peers( - &self, - peers: &[dashcore::network::address::AddrV2Message], - ) -> StorageResult<()> { - let peers_file = self.peers_data_file(); - - let mut buffer = Vec::new(); - - for item in peers.iter() { - item.consensus_encode(&mut buffer) - .map_err(|e| StorageError::WriteFailed(format!("Failed to encode peer: {}", e)))?; - } - - let peers_file_parent = peers_file - .parent() - .ok_or(StorageError::NotFound("peers_file doesn't have a parent".to_string()))?; - - tokio::fs::create_dir_all(peers_file_parent).await?; - - atomic_write(&peers_file, &buffer).await?; - - Ok(()) - } - - async fn load_peers(&self) -> StorageResult> { - let peers_file = self.peers_data_file(); - - if !fs::try_exists(&peers_file).await? { - return Ok(Vec::new()); - }; - - let peers = tokio::task::spawn_blocking(move || { - let file = File::open(&peers_file)?; - let mut reader = BufReader::new(file); - - let mut peers = Vec::new(); - - loop { - match AddrV2Message::consensus_decode(&mut reader) { - Ok(peer) => peers.push(peer), - Err(encode::Error::Io(ref e)) - if e.kind() == std::io::ErrorKind::UnexpectedEof => - { - break - } - Err(e) => { - return Err(StorageError::ReadFailed(format!("Failed to decode peer: {e}"))) - } - } - } - Ok(peers) - }) - .await - .map_err(|e| StorageError::ReadFailed(format!("Failed to load peers: {e}")))??; - - Ok(peers) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use dashcore::network::address::{AddrV2, AddrV2Message}; - use dashcore::network::constants::ServiceFlags; - use tempfile::TempDir; - - #[tokio::test] - async fn test_persistent_peer_storage_save_load() { - let temp_dir = TempDir::new().expect("Failed to create temporary directory for test"); - let store = PersistentPeerStorage::open(temp_dir.path()) - .await - .expect("Failed to open persistent peer storage"); - - // Create test peer messages - let addr: std::net::SocketAddr = - "192.168.1.1:9999".parse().expect("Failed to parse test address"); - let msg = AddrV2Message { - time: 1234567890, - services: ServiceFlags::NETWORK, - addr: AddrV2::Ipv4( - addr.ip().to_string().parse().expect("Failed to parse IPv4 address"), - ), - port: addr.port(), - }; - - store.save_peers(&[msg]).await.expect("Failed to save peers in test"); - - let loaded = store.load_peers().await.expect("Failed to load peers in test"); - assert_eq!(loaded.len(), 1); - assert_eq!(loaded[0].socket_addr().unwrap(), addr); - } - - #[tokio::test] - async fn test_persistent_peer_storage_empty() { - let temp_dir = TempDir::new().expect("Failed to create temporary directory for test"); - let store = PersistentPeerStorage::open(temp_dir.path()) - .await - .expect("Failed to open persistent peer storage"); - - // Load from non-existent file - let loaded = store.load_peers().await.expect("Failed to load peers from empty store"); - assert!(loaded.is_empty()); - } -} diff --git a/dash-spv/tests/dashd_sync/setup.rs b/dash-spv/tests/dashd_sync/setup.rs index b328ae906..250bf9ebd 100644 --- a/dash-spv/tests/dashd_sync/setup.rs +++ b/dash-spv/tests/dashd_sync/setup.rs @@ -1,6 +1,5 @@ use dash_spv::client::config::MempoolStrategy; use dash_spv::network::NetworkEvent; -use dash_spv::storage::{PeerStorage, PersistentPeerStorage, PersistentStorage}; use dash_spv::test_utils::{ create_test_wallet, init_test_logging, next_unused_receive_address, retain_test_dir, DashdTestContext, TestChain, TestEventHandler, @@ -11,8 +10,6 @@ use dash_spv::{ sync::{ProgressPercentage, SyncEvent, SyncProgress}, LoggingGuard, Network, }; -use dashcore::network::address::AddrV2Message; -use dashcore::network::constants::ServiceFlags; use dashcore::Txid; use key_wallet::wallet::managed_wallet_info::wallet_info_interface::WalletInfoInterface; use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; @@ -316,20 +313,16 @@ fn create_test_config(storage_path: PathBuf, peer_addr: std::net::SocketAddr) -> config } -/// Create test client config with no explicit peers (non-exclusive mode). +/// Create a non-exclusive test config (`restrict_to_configured_peers` stays false). /// -/// The peer address is seeded into the peer store on disk so the client -/// discovers it through the normal peer discovery path. -pub(super) async fn create_non_exclusive_test_config( +/// dashd is supplied as a discoverable peer so the client can (re)connect through +/// the discovery path after a disconnect. Regtest has no DNS seeds, so without an +/// injected peer the discovery path would have no source. +pub(super) fn create_non_exclusive_test_config( storage_path: PathBuf, peer_addr: std::net::SocketAddr, ) -> ClientConfig { - let config = ClientConfig::regtest().with_storage_path(storage_path).without_masternodes(); - // Seed the peer store so the client can discover our dashd node - let peer_store = PersistentPeerStorage::open(config.storage_path.clone()) - .await - .expect("Failed to open peer storage"); - let msg = AddrV2Message::new(peer_addr, ServiceFlags::NETWORK); - peer_store.save_peers(&[msg]).await.expect("Failed to seed peer store"); + let mut config = ClientConfig::regtest().with_storage_path(storage_path).without_masternodes(); + config.add_peer(peer_addr); config } diff --git a/dash-spv/tests/dashd_sync/tests_disconnect.rs b/dash-spv/tests/dashd_sync/tests_disconnect.rs index c98c2eee0..1bd8e9c04 100644 --- a/dash-spv/tests/dashd_sync/tests_disconnect.rs +++ b/dash-spv/tests/dashd_sync/tests_disconnect.rs @@ -24,20 +24,19 @@ async fn test_sync_with_peer_disconnection() { /// Verify sync completes in non-exclusive mode despite peer disconnections. /// -/// Unlike `test_sync_with_peer_disconnection` which uses exclusive mode (explicit -/// peers), this test uses non-exclusive mode where the peer is discovered via the -/// seeded peer store. The reconnection path goes through the normal peer discovery -/// mechanism (known addresses + DNS fallback) instead of the exclusive peer list. +/// Unlike `test_sync_with_peer_disconnection` which uses exclusive mode +/// (`restrict_to_configured_peers`), this test runs in non-exclusive mode: dashd is +/// supplied as a discoverable peer and the client (re)connects through the discovery +/// path rather than a restricted, fixed peer list. #[tokio::test] async fn test_sync_with_peer_disconnection_non_exclusive() { let Some(ctx) = TestContext::new(TestChain::Full).await else { return; }; - // Create non-exclusive config: no explicit peers, dashd seeded in peer store + // Non-exclusive config: dashd supplied as a discoverable peer (no peer store). let non_exclusive_config = - create_non_exclusive_test_config(ctx.storage_dir.path().to_path_buf(), ctx.dashd.addr) - .await; + create_non_exclusive_test_config(ctx.storage_dir.path().to_path_buf(), ctx.dashd.addr); let num_disconnects = 3; let client_handle = From 515886c187f3f2657267511ada3bbc55813f6de0 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 11:52:32 +0000 Subject: [PATCH 07/25] removed mem and perf signals --- dash-spv-bench/src/main.rs | 4 - dash-spv/src/network/manager.rs | 10 +- dash-spv/src/network/peer.rs | 2 - dash-spv/src/sync/filters/batch.rs | 4 - dash-spv/src/sync/filters/batch_tracker.rs | 5 - dash-spv/src/sync/filters/manager.rs | 73 -------- dash-spv/src/sync/filters/pipeline.rs | 22 --- dash-spv/src/sync/filters/sync_manager.rs | 8 - dash-spv/src/timer.rs | 200 ++++++--------------- 9 files changed, 56 insertions(+), 272 deletions(-) diff --git a/dash-spv-bench/src/main.rs b/dash-spv-bench/src/main.rs index d3c6c73d6..912ae4e6b 100644 --- a/dash-spv-bench/src/main.rs +++ b/dash-spv-bench/src/main.rs @@ -28,9 +28,6 @@ fn env_or(key: &str, default: &str) -> String { #[tokio::main] async fn main() -> Result<()> { - // Anchor the timeline at process start so `tlog!` timestamps read as time-since-launch. - dash_spv::timer::init(); - // The progress bars own the bottom lines of the terminal, so the logs have to // go through them (they suspend the bars, print, and let them redraw) — writing // to stderr directly would tear the bars apart. @@ -165,7 +162,6 @@ async fn main() -> Result<()> { let _ = tokio::time::timeout(timeout, handler.wait_done()).await; let m = handler.snapshot(); - dash_spv::timer::dump_profile(); let _ = client.shutdown().await; run_handle.abort(); diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index c77e612eb..69d0e3c0b 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -981,7 +981,6 @@ fn spawn_pump( let mut recv_by_peer: HashMap = HashMap::new(); let mut total_recv: u64 = 0; loop { - let t_idle = std::time::Instant::now(); let event = tokio::select! { _ = shutdown.cancelled() => break, ev = inbound.recv() => match ev { @@ -989,10 +988,8 @@ fn spawn_pump( None => break, }, }; - crate::timer::P_PUMP_IDLE.add(t_idle.elapsed()); match event { PeerEvent::Message(addr, msg) => { - let t_busy = std::time::Instant::now(); *recv_by_peer.entry(addr).or_insert(0) += 1; total_recv += 1; if total_recv.is_multiple_of(250_000) { @@ -1001,8 +998,7 @@ fn spawn_pump( dist.sort_by_key(|(_, c)| std::cmp::Reverse(*c)); tracing::info!( target: "filt_depth", - "[t+{}ms] recv balance ({} peers, {} total): {:?}", - crate::timer::elapsed_ms(), + "recv balance ({} peers, {} total): {:?}", dist.len(), total_recv, dist, @@ -1019,8 +1015,7 @@ fn spawn_pump( .collect(); tracing::info!( target: "filt_depth", - "[t+{}ms] peer latency: {:?}", - crate::timer::elapsed_ms(), + "peer latency: {:?}", lat, ); } @@ -1059,7 +1054,6 @@ fn spawn_pump( }); } } - crate::timer::P_PUMP_BUSY.add(t_busy.elapsed()); } PeerEvent::Disconnected(addr) => { let remaining = { diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 4511d4ff9..0ba80df3b 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -449,12 +449,10 @@ fn spawn_reader( ) { tokio::spawn(async move { loop { - let t_recv = std::time::Instant::now(); let next = tokio::select! { _ = shutdown.cancelled() => break, next = reader.next() => next, }; - crate::timer::P_READ_RECV.add(t_recv.elapsed()); match next { None => break, Some(Err(e)) => { diff --git a/dash-spv/src/sync/filters/batch.rs b/dash-spv/src/sync/filters/batch.rs index 80a5aae6d..73ca101fe 100644 --- a/dash-spv/src/sync/filters/batch.rs +++ b/dash-spv/src/sync/filters/batch.rs @@ -102,10 +102,6 @@ impl FiltersBatch { } self.filters_arc.clone().expect("just set") } - /// Bytes held by the sealed `Arc` snapshot (0 if it has not been built). - pub(super) fn filters_arc_bytes(&self) -> usize { - self.filters_arc.as_ref().map(|a| a.iter().map(|(_, f)| f.content.len()).sum()).unwrap_or(0) - } /// Returns whether this batch is verified (filters verified against their headers). pub(super) fn verified(&self) -> bool { self.verified diff --git a/dash-spv/src/sync/filters/batch_tracker.rs b/dash-spv/src/sync/filters/batch_tracker.rs index c657bb1e4..bd43eacd1 100644 --- a/dash-spv/src/sync/filters/batch_tracker.rs +++ b/dash-spv/src/sync/filters/batch_tracker.rs @@ -51,11 +51,6 @@ impl BatchTracker { pub(super) fn end_height(&self) -> u32 { self.end_height } - /// Number of filters received in this batch. - /// TEMPORARY: filters buffered in this (possibly incomplete) batch. - pub(super) fn buffered(&self) -> (usize, usize) { - (self.filters.len(), self.filters.values().map(|f| f.content.len()).sum()) - } pub(super) fn received(&self) -> u32 { self.received.len() as u32 diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index 6529aef4f..e6f1f5b00 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -416,18 +416,6 @@ async fn match_filters_multi( result } -/// Folds the elapsed time into `P_SCAN` on drop, so `scan_batch`'s several early -/// returns all get accounted for. -struct ScanGuard(std::time::Instant); -impl Drop for ScanGuard { - fn drop(&mut self) { - crate::timer::P_SCAN.add(self.0.elapsed()); - } -} -fn scopeguard_scan(t: std::time::Instant) -> ScanGuard { - ScanGuard(t) -} - /// Snapshot of a behind wallet's compact-filter query inputs for a batch scan. struct WalletScanState { /// The wallet these inputs belong to. @@ -518,8 +506,6 @@ pub struct FiltersManager< /// reach, so they arrive around the same time as these do — keeping them here /// means the verification never has to read back what we were just handed. pub(super) filter_headers: BTreeMap, - /// Last time the throttled MEM debug line was emitted (once per 2s). - last_mem_log: Option, /// Cached wallet birth (`earliest_required_height`), refreshed each /// `start_download`. Lets the BlockProcessed handler roll a wallet's frontier /// back to birth WITHOUT taking a wallet lock — doing that read inside the @@ -585,7 +571,6 @@ impl SyncResult> { let mut events = Vec::new(); - // MEM (throttled): what the filters manager is actually holding. Sums the - // real filter bytes rather than counting structures, so the numbers can be - // compared against the process RSS directly. - { - let now = std::time::Instant::now(); - let due = self - .last_mem_log - .is_none_or(|last| now.duration_since(last) >= std::time::Duration::from_secs(2)); - if due { - self.last_mem_log = Some(now); - let pend_bytes: usize = self - .pending_batches - .iter() - .flat_map(|b| b.filters().values()) - .map(|f| f.content.len()) - .sum(); - let act_bytes: usize = self - .active_batches - .values() - .flat_map(|b| b.filters().values()) - .map(|f| f.content.len()) - .sum(); - let arc_bytes: usize = - self.active_batches.values().map(|b| b.filters_arc_bytes()).sum(); - let (trk_batches, trk_filters, trk_bytes) = - self.filter_pipeline.buffered_in_trackers(); - tracing::info!( - target: "dash_spv::sync::filters", - "MEM incomplete={} batches ({} filters, {:.0} MB) | pending={} batches ({:.0} MB) | active={} batches (map {:.0} MB + snapshot {:.0} MB) | fheader_cache={} ({:.0} MB)", - trk_batches, - trk_filters, - trk_bytes as f64 / 1e6, - self.pending_batches.len(), - pend_bytes as f64 / 1e6, - self.active_batches.len(), - act_bytes as f64 / 1e6, - arc_bytes as f64 / 1e6, - self.filter_headers.len(), - (self.filter_headers.len() * 36) as f64 / 1e6, - ); - } - } - // Phase 0: fold the scripts that freshly processed blocks derived into the // shared append-only log. self.fold_derived_scripts(); @@ -1373,7 +1311,6 @@ impl SyncResult> { - let t_bulk = std::time::Instant::now(); let mut events = Vec::new(); let log_len = self.deferred.len(); if log_len == 0 || self.total_pending_blocks() > 0 { @@ -1421,7 +1358,6 @@ impl SyncResult> { - let t_commit = std::time::Instant::now(); let mut events = Vec::new(); loop { @@ -1531,7 +1466,6 @@ impl> = HashMap::new(); @@ -1540,7 +1474,6 @@ impl SyncResult> { - let t_scan = std::time::Instant::now(); let mut events = Vec::new(); let starts: Vec = self @@ -1653,7 +1584,6 @@ impl SyncResult> { - let t_scan = std::time::Instant::now(); - let _g = scopeguard_scan(t_scan); let mut events = Vec::new(); let (batch_end, filters_empty) = { diff --git a/dash-spv/src/sync/filters/pipeline.rs b/dash-spv/src/sync/filters/pipeline.rs index d77ee1076..0c4680d33 100644 --- a/dash-spv/src/sync/filters/pipeline.rs +++ b/dash-spv/src/sync/filters/pipeline.rs @@ -128,24 +128,6 @@ impl FiltersPipeline { } } - /// TEMPORARY: filters sitting in incomplete batches. A batch is only consumed once - /// ALL of its ~1000 filters have arrived, so partially-filled trackers hold filters - /// that nothing downstream can touch yet. - pub(super) fn buffered_in_trackers(&self) -> (usize, usize, usize) { - let mut batches = 0; - let mut filters = 0; - let mut bytes = 0; - for t in self.batch_trackers.values() { - let (n, b) = t.buffered(); - if n > 0 { - batches += 1; - filters += n; - bytes += b; - } - } - (batches, filters, bytes) - } - /// Start the response timeout for a batch the router just put on the wire. pub(super) fn mark_on_flight(&mut self, start_height: u32) { self.coordinator.mark_on_flight(&[start_height]); @@ -235,14 +217,10 @@ impl FiltersPipeline { filter_data: &[u8], ) -> Option { // Find which batch this filter belongs to - let tf = std::time::Instant::now(); let batch_start = self.find_batch_for_height(height)?; - crate::timer::P_FIND_BATCH.add(tf.elapsed()); let tracker = self.batch_trackers.get_mut(&batch_start)?; - let ti = std::time::Instant::now(); tracker.insert_filter(height, block_hash, filter_data); - crate::timer::P_INSERT_FILTER.add(ti.elapsed()); self.filters_received += 1; self.highest_received = self.highest_received.max(height); diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 2ddb9990a..4d568adba 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -119,10 +119,8 @@ impl< }; // Find height for this filter - let t0 = std::time::Instant::now(); let height = self.header_storage.read().await.get_header_height_by_hash(&cfilter.block_hash).await?; - crate::timer::P_HEIGHT_LOOKUP.add(t0.elapsed()); let Some(h) = height else { tracing::warn!( @@ -138,10 +136,8 @@ impl< }; // Buffer filter in pipeline - let t1 = std::time::Instant::now(); let batch_completed = self.filter_pipeline.receive_with_data(h, cfilter.block_hash, &cfilter.filter); - crate::timer::P_RECV_DATA.add(t1.elapsed()); // A completed batch == one `getcfilters` request fully answered: free that // peer's in-flight unit (the reader skips per-`cfilter` decrements). @@ -150,15 +146,11 @@ impl< } // Send more requests if there are free slots - let t2 = std::time::Instant::now(); let header_storage = self.header_storage.read().await; self.filter_pipeline.send_pending(network, &*header_storage).await?; drop(header_storage); - crate::timer::P_SEND_PENDING.add(t2.elapsed()); - let t3 = std::time::Instant::now(); let events = self.store_and_match_batches().await?; - crate::timer::P_STORE_MATCH.add(t3.elapsed()); Ok(events) } diff --git a/dash-spv/src/timer.rs b/dash-spv/src/timer.rs index 5f4f74a8f..555052831 100644 --- a/dash-spv/src/timer.rs +++ b/dash-spv/src/timer.rs @@ -1,182 +1,90 @@ -//! Process-relative timeline timestamps for pinpointing where sync wall-clock time goes. +//! Lightweight timing primitives for ad-hoc profiling. //! -//! One process-start [`Instant`] plus the [`tlog!`](crate::tlog) macro: drop `tlog!("...")` at -//! interesting points (before/after a network round-trip, before/after hashing, storage writes, …) -//! and every line is prefixed with the elapsed time since the timeline was anchored. The gap -//! between two consecutive timeline lines is the time spent in between — so you can add marks -//! incrementally and bisect where the time is going. +//! This module is intentionally minimal and deliberately has **no callers** in +//! the normal build. It is scaffolding for an AI agent to run A/B experiments +//! between different optimization strategies: wire up the specific measurements a +//! hypothesis needs, compare the numbers across strategies, then rip the +//! instrumentation back out once the question is answered — leaving the hot paths +//! clean again. //! -//! # Usage +//! Typical use: //! ```ignore -//! dash_spv::timer::init(); // once, at the top of main() (optional; otherwise anchored lazily) -//! tlog!("about to send GetHeaders (seg {})", seg_id); -//! // ... network round-trip ... -//! tlog!("got {} headers", headers.len()); +//! use crate::timer::Prof; +//! +//! // One static per hot path under investigation. +//! static P_SCAN: Prof = Prof::new("filters.scan_batch"); +//! +//! // On the hot path: +//! let t = std::time::Instant::now(); +//! // ... work to measure ... +//! P_SCAN.add(t.elapsed()); +//! +//! // Somewhere observable (e.g. on shutdown), read it back: +//! let (name, calls, total, mean) = P_SCAN.report(); +//! tracing::info!("{name}: {calls} calls, {total:?} total, {mean:?} mean"); //! ``` -//! Timeline lines use the `timeline` tracing target at INFO — enable them with `timeline=info` -//! in `RUST_LOG` / the subscriber's env filter. +//! +//! Use [`elapsed_ms`] for `[t+Nms]`-style relative timestamps when correlating +//! log lines across a run. +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::LazyLock; use std::time::{Duration, Instant}; -/// The instant the timeline started. Anchored on first access; call [`init`] early in `main` to -/// anchor it at process start rather than at the first [`elapsed`]/`tlog!`. +/// Process-start instant, initialized lazily on first use (or eagerly via +/// [`init`]). Anchors the relative timeline that [`elapsed`]/[`elapsed_ms`] read. pub static START: LazyLock = LazyLock::new(Instant::now); -/// Anchor the timeline now (idempotent). Call once at the top of `main` so `t+0` == process start. +/// Anchor the timeline at process start. Optional — [`START`] initializes lazily +/// on first read; call this early if you want the anchor pinned before any work. pub fn init() { LazyLock::force(&START); } -/// Time elapsed since the timeline started. -#[inline] +/// Time elapsed since the timeline was anchored (see [`init`]). pub fn elapsed() -> Duration { START.elapsed() } -/// Milliseconds elapsed since the timeline started (what [`tlog!`](crate::tlog) prints). -#[inline] +/// Time elapsed since the timeline was anchored, in whole milliseconds. Handy for +/// `[t+{}ms]` log prefixes that correlate events across a run. pub fn elapsed_ms() -> u128 { START.elapsed().as_millis() } -use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; - -/// A named time accumulator for micro-profiling a hot path called too often to `tlog!` per call -/// (e.g. once per cfilter message). Sum time with [`Prof::add`], print all with [`dump_profile`]. +/// A cheap, lock-free accumulating profiler: fold elapsed durations into a +/// running call count and total with [`add`](Prof::add), read them back with +/// [`report`](Prof::report). Declare one as a `static` per measured path. pub struct Prof { - pub name: &'static str, - ns: AtomicU64, - calls: AtomicU64, + name: &'static str, + count: AtomicU64, + total_ns: AtomicU64, } impl Prof { pub const fn new(name: &'static str) -> Self { Self { name, - ns: AtomicU64::new(0), - calls: AtomicU64::new(0), + count: AtomicU64::new(0), + total_ns: AtomicU64::new(0), } } - /// Add one sample. - #[inline] + + /// Fold one measured duration into the running total. pub fn add(&self, d: Duration) { - self.ns.fetch_add(d.as_nanos() as u64, Relaxed); - self.calls.fetch_add(1, Relaxed); + self.count.fetch_add(1, Ordering::Relaxed); + self.total_ns.fetch_add(d.as_nanos() as u64, Ordering::Relaxed); } -} - -// Buckets for the per-cfilter-message hot path (filters/sync_manager.rs::handle_message). -pub static P_HEIGHT_LOOKUP: Prof = Prof::new("filt.height_lookup"); -pub static P_RECV_DATA: Prof = Prof::new("filt.receive_with_data"); -pub static P_SEND_PENDING: Prof = Prof::new("filt.send_pending"); -pub static P_STORE_MATCH: Prof = Prof::new("filt.store_and_match"); - -// Filters TAIL path (runs off the cfilter hot path, driven by BlockProcessed): where the -// gap-limit rescan cascade actually spends its wall clock. SCAN = first pass over a batch's -// filters; BULK = the lookahead match of the derived-script log across all open batches; -// HEAD = the commit head's own catch-up match; DISK = filter writes in the sequential store; -// COMMIT = the whole commit loop. If SCAN+BULK+HEAD << tail duration, we are waiting on the -// matched-block downloads, not on CPU. -pub static P_SCAN: Prof = Prof::new("filt.scan_batch"); -pub static P_BULK: Prof = Prof::new("filt.bulk_rescan"); -pub static P_HEAD: Prof = Prof::new("filt.head_rescan"); -pub static P_DISK: Prof = Prof::new("filt.disk_store"); -pub static P_VERIFY: Prof = Prof::new("filt.verify"); -pub static P_COMMIT: Prof = Prof::new("filt.commit_loop"); -// Arrival path: time to lock the shared Arc> and dispatch each message. -// If per-call time balloons with more peers, that Mutex is the ~70k/s serialization point. -pub static P_NET_DISPATCH: Prof = Prof::new("net.dispatch(lock)"); - -// network pump (single task draining PeerEvents): BUSY = per-message dispatch work -// (subscriber lock + clone + channel send); IDLE = time blocked in inbound.recv().await. -// IDLE >> BUSY => the pump is starved (readers/peers slow), not the bottleneck. -pub static P_PUMP_BUSY: Prof = Prof::new("pump.dispatch_busy"); -pub static P_PUMP_IDLE: Prof = Prof::new("pump.idle_wait"); - -// receive_with_data breakdown: FIND = linear scan over batch_trackers to locate -// the height's batch; INSERT = BlockFilter::new (clones bytes) + hashmap inserts. -pub static P_FIND_BATCH: Prof = Prof::new("filt.find_batch"); -pub static P_INSERT_FILTER: Prof = Prof::new("filt.insert_filter"); - -// Reader loop (network/manager.rs) per-iteration breakdown. RLOCK+WLOCK are the two peer-lock -// acquisitions per message; RECV is the receive_message/select; MSG vs IDLE count how often an -// iteration yielded a message vs timed out waiting (IDLE-heavy => peer/network-bound, not reader). -pub static P_READ_RLOCK: Prof = Prof::new("read.rlock"); -pub static P_READ_WLOCK: Prof = Prof::new("read.wlock"); -pub static P_READ_RECV: Prof = Prof::new("read.recv/select"); -pub static P_READ_MSG: Prof = Prof::new("read.got_msg"); -pub static P_READ_IDLE: Prof = Prof::new("read.idle_timeout"); - -// Lock contention probes for the cfilter phase. -// READER_HOLD: how long the reader holds peer.write() across receive_message/select (if long, it -// blocks the sender which needs the same lock). SENDER_WAIT: how long send_message_to_peer waits -// to acquire peer.write() (high => reader is hogging it). HM_HDR: per-cfilter header_storage.read -// acquire wait in handle_message. DISPATCH_WAIT: message_dispatcher Mutex acquire wait. -pub static P_READER_HOLD: Prof = Prof::new("reader.peer_write_HOLD"); -pub static P_SENDER_WAIT: Prof = Prof::new("sender.peer_write_WAIT"); -pub static P_HM_HDR: Prof = Prof::new("handle_msg.hdr_read_wait"); -pub static P_DISPATCH_WAIT: Prof = Prof::new("dispatch.lock_WAIT"); - -/// Log all profiling accumulators (call once at end of run). Uses the `timeline` target. -pub fn dump_profile() { - // eprintln so it prints even when the `timeline` target is filtered out (so we can profile with - // the high-volume per-message marks disabled). - for p in [ - &P_HEIGHT_LOOKUP, - &P_RECV_DATA, - &P_SEND_PENDING, - &P_STORE_MATCH, - &P_SCAN, - &P_BULK, - &P_HEAD, - &P_DISK, - &P_VERIFY, - &P_COMMIT, - &P_FIND_BATCH, - &P_INSERT_FILTER, - &P_NET_DISPATCH, - &P_PUMP_BUSY, - &P_PUMP_IDLE, - &P_READ_RLOCK, - &P_READ_WLOCK, - &P_READ_RECV, - &P_READ_MSG, - &P_READ_IDLE, - &P_READER_HOLD, - &P_SENDER_WAIT, - &P_HM_HDR, - &P_DISPATCH_WAIT, - ] { - let ns = p.ns.load(Relaxed); - let c = p.calls.load(Relaxed); - if c > 0 { - eprintln!( - "[t+{:>7}ms] PROF {:22} calls={:>8} total={:>6}ms per={:>7.3}us", - elapsed_ms(), - p.name, - c, - ns / 1_000_000, - ns as f64 / c as f64 / 1000.0 - ); - } + /// `(name, calls, total, mean)` — the accumulated stats for reporting. + pub fn report(&self) -> (&'static str, u64, Duration, Duration) { + let count = self.count.load(Ordering::Relaxed); + let total = Duration::from_nanos(self.total_ns.load(Ordering::Relaxed)); + let mean = if count > 0 { + total / count as u32 + } else { + Duration::ZERO + }; + (self.name, count, total, mean) } } - -/// Log a message prefixed with the elapsed time since the process timeline started, e.g. -/// `[t+ 1234ms] sent GetHeaders`. Uses the `timeline` tracing target at INFO level. -/// -/// Cheap to leave in place, easy to add more of — the point is to sprinkle these around hot spots -/// and read off the gaps between consecutive lines. -#[macro_export] -macro_rules! tlog { - ($($arg:tt)*) => { - ::tracing::info!( - target: "timeline", - "[t+{:>7}ms] {}", - $crate::timer::elapsed_ms(), - format_args!($($arg)*) - ) - }; -} From 5ded5d3358cf1dbe81db3a0d3fc7891e0c7e1617 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 12:41:06 +0000 Subject: [PATCH 08/25] correctly read tip at sync start --- dash-spv/src/network/manager.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 69d0e3c0b..f06721bb9 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -259,6 +259,7 @@ impl PeerNetworkManager { bytes.clone(), global_cap.clone(), events_tx.clone(), + best_tip.clone(), ); spawn_bandwidth_controller( @@ -315,7 +316,7 @@ impl PeerNetworkManager { ) .await; - self.best_tip.store(best_tip, Ordering::Relaxed); + self.best_tip.fetch_max(best_tip, Ordering::Relaxed); // Announce each peer individually before the summary. Managers that track the peer // set (the mempool, which must send `filterload` to turn on transaction relay) build @@ -435,6 +436,7 @@ fn spawn_router( bytes: Arc, global_cap: Arc, events: broadcast::Sender, + best_tip: Arc, ) -> JoinHandle<()> { tokio::spawn(async move { loop { @@ -467,6 +469,7 @@ fn spawn_router( &shutdown, &bytes, &events, + &best_tip, ADD_PEERS_BATCH, MAX_CONNECTED_PEERS, ) @@ -818,7 +821,8 @@ fn spawn_reconnector( let before = connected.lock().await.len(); add_peers( - &connected, &others, &inbound, &shutdown, &bytes, &events, deficit, max_peers, + &connected, &others, &inbound, &shutdown, &bytes, &events, &best_tip, deficit, + max_peers, ) .await; let after = connected.lock().await.len(); @@ -849,6 +853,7 @@ async fn add_peers( shutdown: &CancellationToken, bytes: &Arc, events: &broadcast::Sender, + best_tip: &Arc, batch: usize, max_peers: usize, ) { @@ -868,6 +873,11 @@ async fn add_peers( if let Ok(peer) = candidate.connect(inbound.clone(), shutdown.clone(), bytes.clone()).await { let addr = peer.addr(); + // Keep `best_tip` current the moment a peer arrives: a newcomer may + // advertise a higher chain tip than the initial probe saw, and the sync + // managers seed their target height from `best_tip` on `PeersUpdated`, so + // it must be right before that event fires — not only after the probe. + best_tip.fetch_max(peer.version().start_height.max(0) as u32, Ordering::Relaxed); connected.lock().await.push((peer, State {})); let _ = events.send(NetworkEvent::PeerConnected(addr)); added += 1; From 3e4fa1c57abec05cf6896d7071b2d5bc71226eb8 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 13:31:20 +0000 Subject: [PATCH 09/25] fix(dash-spv): stop infinite filter rescan when wallet birth is below the sync checkpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge of dev brought in the checkpoint floor (a birth-0 HD wallet now anchors mainnet sync at the 200000 checkpoint instead of genesis, since no HD wallet has transactions before it). That exposed a latent bug in the filters manager's stale-wallet check: it compared each wallet's raw `synced_height` against `committed_height`. A fresh wallet reports `synced_height = 0`, but a checkpoint sync sets `committed_height = checkpoint - 1` (199999), so the wallet read as perpetually "behind" — every tick reset and restarted the scan, an infinite loop that starved block/filter-header processing and stalled the whole sync at ~12%. No filters exist below the sync floor, so a wallet cannot be behind there: clamp its effective scanned baseline to `header_start_height` before the comparison. A genuinely late-added wallet (synced within the synced range) still triggers a rescan; a fresh below-floor wallet no longer does. Verified on a real mainnet bench: headers and filter headers now reach 100% (were stuck at ~12%), zero "restarting scan" loops. Co-Authored-By: Claude Opus 4.8 --- dash-spv/src/sync/filters/sync_manager.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 4d568adba..28e3737e3 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -253,11 +253,23 @@ impl< // wallets are not re-scanned from scratch. if matches!(self.state(), SyncState::Syncing | SyncState::Synced | SyncState::WaitForEvents) { + // No filters exist below the sync floor (the checkpoint the chain is + // anchored at), so a wallet cannot be "behind" there: its effective + // scanned baseline is the floor, not its raw `synced_height`. Without + // this clamp a fresh wallet (synced_height 0) whose birth is below a + // checkpoint we started from reads as perpetually behind committed + // (= floor - 1), so every tick resets and restarts the scan — an + // infinite rescan loop that starves header processing and stalls sync. + let header_start = + self.header_storage.read().await.get_start_height().await.unwrap_or(0); let committed = self.progress.committed_height(); let wallet_read = self.wallet.read().await; let behind = wallet_read.wallets_behind(committed); - let stale_min_synced = - behind.iter().map(|id| wallet_read.wallet_synced_height(id)).min(); + let stale_min_synced = behind + .iter() + .map(|id| wallet_read.wallet_synced_height(id).max(header_start)) + .filter(|&synced| synced < committed) + .min(); drop(wallet_read); if let Some(stale_min_synced) = stale_min_synced { tracing::info!( From e6b667cadd563c0966ff1da0158e396fa3fa3f08 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 13:53:54 +0000 Subject: [PATCH 10/25] fix(dash-spv): start filter bodies at the first verifiable height on in-memory checkpoint sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second checkpoint-sync stall (after the rescan loop): the filters phase spun without ever committing or matching — committed stuck at the checkpoint, downloaded/matched 0 — because verifying the first filter batch failed with "Missing filter header at height ". Filter-header sync anchors the chain differently per mode. The storage path stores the anchor (`cfheaders.previous_filter_header`) at `header_start - 1`, so filter bodies verify from `header_start`. The in-memory path (which runs on a fresh checkpoint sync) anchors it at `header_start` itself, so filter_header [header_start - 1] never exists — yet the filters manager still tried to download and verify a body at `header_start`, which needs exactly that missing header. Raise the filters download/scan floor to `filter_header_start + 1` when the anchor sits above genesis, so the first body downloaded is the lowest one whose predecessor filter header is actually known. No-op for genesis (anchor at 0, predecessor is the all-zeros sentinel) and for the storage/resume path (anchor at header_start - 1, floor already equals header_start). Verified on a real mainnet checkpoint sync: zero "Missing filter header" errors, filters commit and match (blocks download with relevant transactions), phases reach 100%. Integration: lib 184/184, dashd basic 3/3, restart 4/4. Co-Authored-By: Claude Opus 4.8 --- dash-spv/src/sync/filters/manager.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index 2fc576b89..fa7f267ea 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -829,9 +829,23 @@ impl 0 { + header_start_height = header_start_height.max(fhs + 1); + } + } + // Calculate scan start (where we need to start processing) // Must be at least header_start_height for checkpoint-based sync let committed_start = if wallet_committed_height > 0 { From c23c2ed36b42e038ecee9ed6c46a1437eb6cc985 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 14:47:19 +0000 Subject: [PATCH 11/25] fix(dash-spv): drain in-flight requests before retiring displaced peers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intermittently a mainnet sync stalled at the very start of header sync with only the first segment stored. Cause: at startup the reconnector races the probe — it connects peers from the discoverer and the sync sends them the initial per-segment GetHeaders, then `probe_and_select` settles the final peer set and replaced `connected` wholesale, dropping those peers mid-request. The stranded requests only recovered on the pipelines' own slow timeout, and occasionally a run never remounted (the surviving peers were slow/quiet), stalling all segments above the first. Retire displaced peers by draining instead of dropping: if a peer has requests in flight, keep its connection alive in the background (its reader keeps delivering responses and decrementing `in_flight`) and close it only once it has drained, capped at STALE_REQUEST so a peer that never drains is still reaped. A peer with nothing in flight is closed immediately as before. Bonus: previously these dropped peers leaked their reader tasks until global shutdown; now they close cleanly. Verified: 8/8 back-to-back mainnet syncs completed with zero stalls (a prior run of 5 stalled 1); the one run that hit the churn scenario recovered via the drain+retry instead of hanging. Co-Authored-By: Claude Opus 4.8 --- dash-spv/src/network/manager.rs | 40 ++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index f06721bb9..b1d614902 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -47,6 +47,10 @@ const STALL_CHECK: Duration = Duration::from_secs(5); /// free capacity, not to punish peers (whether the peer is dropped is decided /// separately, on whether it has EVER answered). const STALE_REQUEST: Duration = Duration::from_secs(90); + +/// Poll interval for draining a retired peer's in-flight requests (see +/// [`retire_drained`]). The drain is capped at [`STALE_REQUEST`]. +const DRAIN_POLL: Duration = Duration::from_secs(1); use crate::{ network::{ discovery::PeerDiscoverer, @@ -895,6 +899,32 @@ async fn add_peers( } } +/// Retire a peer we are dropping from the active set WITHOUT stranding requests +/// already on the wire to it. During startup the reconnector connects peers and +/// the sync sends them pipeline requests before the probe has settled the final +/// peer set; replacing the set would then drop those peers mid-request, and the +/// stranded requests only recover on the pipelines' own (slow) timeout — +/// occasionally stalling whole header segments for a run. Instead keep the +/// connection alive in the background: its reader keeps delivering responses and +/// decrementing `in_flight`. Close it once it has drained, or after +/// `STALE_REQUEST` (a peer that never drains is dead), whichever comes first. +fn retire_drained(peer: ConnectedPeer, shutdown: CancellationToken) { + if peer.in_flight() == 0 { + peer.close(); + return; + } + tokio::spawn(async move { + let mut waited = Duration::ZERO; + while peer.in_flight() > 0 && waited < STALE_REQUEST { + tokio::select! { + _ = shutdown.cancelled() => return, + _ = tokio::time::sleep(DRAIN_POLL) => waited += DRAIN_POLL, + } + } + peer.close(); + }); +} + async fn probe_and_select( discoverer: &mut PeerDiscoverer, connected: &Mutex>, @@ -972,7 +1002,15 @@ async fn probe_and_select( tip, ); - *connected.lock().await = kept.into_iter().map(|peer| (peer, State {})).collect(); + let new_connected: Vec<(ConnectedPeer, State)> = + kept.into_iter().map(|peer| (peer, State {})).collect(); + // Swap in the probe's selection, but don't strand requests the sync already + // put on the peers the reconnector raced in during startup: drain those in + // the background instead of dropping them here (see `retire_drained`). + let displaced = std::mem::replace(&mut *connected.lock().await, new_connected); + for (peer, _) in displaced { + retire_drained(peer, shutdown.clone()); + } *others.lock().await = backups; tip From c42dabd6652b526debd6438fbede4736b4f75218 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 14:47:26 +0000 Subject: [PATCH 12/25] chore(dash-spv-bench): bump indicatif 0.17 -> 0.18 to drop unmaintained number_prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cargo-deny fails the branch: indicatif 0.17.11 pulls number_prefix 0.4.0, flagged unmaintained (RUSTSEC-2025-0119). indicatif 0.18 replaced number_prefix with unit-prefix, so the bump removes the advisory (and one duplicate windows-sys) from the tree. Drop-in — the bench only uses stable MultiProgress/ProgressBar/ ProgressStyle APIs. (Cargo.lock is gitignored, so CI resolves this fresh.) Co-Authored-By: Claude Opus 4.8 --- dash-spv-bench/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash-spv-bench/Cargo.toml b/dash-spv-bench/Cargo.toml index 776fa96d3..0cf11a8b9 100644 --- a/dash-spv-bench/Cargo.toml +++ b/dash-spv-bench/Cargo.toml @@ -18,7 +18,7 @@ tempfile = "3.0" tracing = "0.1" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } -indicatif = "0.17" +indicatif = "0.18" [[bin]] name = "dash-spv-bench" From 96c796dd3b837a0aba11d9695c7c2c2d3bf3ff86 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 14:52:13 +0000 Subject: [PATCH 13/25] ci: include dash-spv-bench in the excluded ci group --- .github/ci-groups.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ci-groups.yml b/.github/ci-groups.yml index cfd026315..2bd5ed69b 100644 --- a/.github/ci-groups.yml +++ b/.github/ci-groups.yml @@ -30,3 +30,4 @@ excluded: - dash-fuzz # Honggfuzz binary targets, tested by fuzz.yml - masternode-seeds-fetcher # Tooling binary; needs live dashd RPC, exercised by update-masternode-seeds.yml - git-state # Compile-time proc-macro with no tests, exercised via dash-spv + - dash-spv-bench # Benchmarking binary From a976adc7fc551b59cbf7eb0b0f2bf5a99a1d2194 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 14:53:32 +0000 Subject: [PATCH 14/25] fix(dash-spv): keep internal impl guidance off the public SyncManager doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cargo doc` with `-D warnings` failed: the public trait method `SyncManager::handle_network_event` doc-linked `default_handle_network_event`, which is `pub(super)` — a private-intra-doc-link, denied by `rustdoc::private_intra_doc_links`. The delegation advice ("override and defer the rest to the default body; Rust can't call a trait's default body from an override") is guidance for the crate's own sync-manager implementors, not the public API — an external reader can't call the `pub(super)` helper anyway. So drop it from the public method doc, which now just describes behavior, and move the full note onto `default_handle_network_event` itself (a private item, only rendered under `--document-private-items`, where the link resolves cleanly). `default_handle_network_event` stays private — exposing it publicly just to satisfy a link would grow the API surface for no external benefit. Verified: `RUSTDOCFLAGS="-D warnings" cargo doc -p dash-spv --all-features --no-deps` clean. Co-Authored-By: Claude Opus 4.8 --- dash-spv/src/sync/sync_manager.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dash-spv/src/sync/sync_manager.rs b/dash-spv/src/sync/sync_manager.rs index d0fd92d46..bea6d528c 100644 --- a/dash-spv/src/sync/sync_manager.rs +++ b/dash-spv/src/sync/sync_manager.rs @@ -89,7 +89,12 @@ impl std::fmt::Display for NetworkEvent { } } -/// The default `SyncManager::handle_network_event` body, callable from an override. +/// The default [`SyncManager::handle_network_event`] body, callable from an override. +/// +/// A manager that only cares about some `NetworkEvent` variants overrides +/// `handle_network_event` and delegates the rest here. Rust gives no way to call a +/// trait's default body from an override, so the shared logic lives in this free +/// function — the trait's default method just calls it. pub(super) async fn default_handle_network_event( manager: &mut M, event: &NetworkEvent, @@ -219,10 +224,8 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { /// Handle a network event (peer connection changes). /// - /// Default implementation handles state transitions for WaitingForConnections. - /// Managers can override to customize behavior — an override that only cares about - /// some variants should delegate the rest to [`default_handle_network_event`], since - /// Rust gives no way to call a trait's default body from an override. + /// The default body handles state transitions for `WaitingForConnections`. + /// Managers can override this to customize behavior. async fn handle_network_event( &mut self, event: &NetworkEvent, From 6cd7aae3be75b60f27cef409ea0e25b6900f74a5 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Thu, 16 Jul 2026 22:14:06 +0000 Subject: [PATCH 15/25] refactor(dash-spv): network manager cares about download coordination alone --- dash-spv/src/network/manager.rs | 324 +++++++++++++----- dash-spv/src/network/peer.rs | 52 --- dash-spv/src/sync/block_headers/manager.rs | 9 +- dash-spv/src/sync/block_headers/pipeline.rs | 51 ++- .../src/sync/block_headers/segment_state.rs | 58 ++-- .../src/sync/block_headers/sync_manager.rs | 15 +- dash-spv/src/sync/blocks/pipeline.rs | 94 ++--- dash-spv/src/sync/blocks/sync_manager.rs | 23 +- dash-spv/src/sync/download_coordinator.rs | 239 ------------- dash-spv/src/sync/filter_headers/pipeline.rs | 96 ++---- .../src/sync/filter_headers/sync_manager.rs | 18 +- dash-spv/src/sync/filters/pipeline.rs | 152 ++++---- dash-spv/src/sync/filters/sync_manager.rs | 26 +- dash-spv/src/sync/masternodes/pipeline.rs | 70 ++-- dash-spv/src/sync/masternodes/sync_manager.rs | 12 +- dash-spv/src/sync/mod.rs | 1 - dash-spv/src/sync/sync_manager.rs | 24 +- 17 files changed, 493 insertions(+), 771 deletions(-) delete mode 100644 dash-spv/src/sync/download_coordinator.rs diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index b1d614902..706b900dc 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -5,7 +5,7 @@ use std::{ atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}, Arc, }, - time::Duration, + time::{Duration, Instant}, }; use dashcore::network::message::NetworkMessage; @@ -34,22 +34,29 @@ const MAX_CONNECTED_PEERS: usize = 16; /// How often the reconnector checks whether the peer set needs topping up. const RECONNECT_CHECK: Duration = Duration::from_secs(2); -/// How long the router waits on a full-capacity stall before checking whether the -/// "in-flight" requests holding that capacity are actually dead. +/// How long the router sleeps on a full-capacity stall before re-evaluating. It +/// wakes early whenever a response frees a slot or the timeout monitor kicks a +/// peer; this is just a backstop so it never sleeps on a notify that never comes. const STALL_CHECK: Duration = Duration::from_secs(5); -/// A request unanswered for this long has its slot reclaimed: the owning pipeline -/// has long since timed it out and re-queued it elsewhere, so the slot holds -/// nothing. +/// A request unanswered for this long is treated as dead: the timeout monitor +/// re-queues it to another peer and kicks the peer that ignored it. /// -/// Generous, because slow is not dead. Blocks are paced through the same budget, -/// and a 2MB block over a weak link legitimately takes a while — the point is to -/// free capacity, not to punish peers (whether the peer is dropped is decided -/// separately, on whether it has EVER answered). -const STALE_REQUEST: Duration = Duration::from_secs(90); +/// Uniform across request types. Blocks are paced through the same budget and a +/// 2MB block over a weak link legitimately takes a while, but a peer that has +/// answered nothing in 30s is not worth waiting on — the reconnector refills the +/// slot with a fresh peer faster than a dead one would ever recover. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// How often the timeout monitor scans the outstanding-request registry. +const TIMEOUT_CHECK: Duration = Duration::from_secs(1); + +/// How long a retired peer (displaced during startup, see [`retire_drained`]) is +/// kept alive to drain its in-flight responses before being force-closed. +const RETIRE_DRAIN_CAP: Duration = Duration::from_secs(90); /// Poll interval for draining a retired peer's in-flight requests (see -/// [`retire_drained`]). The drain is capped at [`STALE_REQUEST`]. +/// [`retire_drained`]). The drain is capped at [`RETIRE_DRAIN_CAP`]. const DRAIN_POLL: Duration = Duration::from_secs(1); use crate::{ network::{ @@ -66,6 +73,17 @@ use crate::{ type Inbound = (SocketAddr, Arc); type Subscribers = Arc>>>>; +/// Every pipeline request the broker is handling, keyed by its identity, from the +/// moment `send` accepts it until the owning manager reports it answered +/// (`request_answered`) or cancels it (`cancel`). This is the single home of +/// request state: the pipelines hold no download coordinator, they just declare +/// what they want and the broker de-duplicates, paces, times out and retries. +/// +/// - Membership is the de-dup set: `send` ignores a key already present. +/// - `OnWire` entries carry the message so the timeout monitor can re-inject it +/// (retry) after dropping the peer that ignored it. +type Registry = Arc>>; + /// The kinds of peer message a sync manager can subscribe to. Replaces the /// stringly-typed command names: managers declare interest with these variants /// and the pump routes incoming messages by mapping `cmd()` back to one. @@ -129,17 +147,13 @@ pub enum NetworkEvent { }, PeerConnected(SocketAddr), PeerDisconnected(SocketAddr), - /// A pipeline request just left the router for a peer (it is now on the - /// wire). Pipelines use this to start the request's response timeout from the - /// actual send, not from when it was queued (which fires spuriously when the - /// router is backed up). - RequestOnFlight(RequestKey), } -/// Identifies a pipeline request that has been sent, so the owning pipeline can -/// match it to its in-flight entry and start the timeout. One variant per -/// router-paced request type. -#[derive(Clone, Debug)] +/// Identifies a pipeline request the network manager tracks from send to +/// response, so it can time the request out and re-queue it (and drop the peer +/// that ignored it). One variant per router-paced request type. Used as a map +/// key in the outstanding-request registry, hence `Hash`/`Eq`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum RequestKey { /// `getheaders` — keyed by the locator's first hash (the segment tip). Headers(dashcore::BlockHash), @@ -147,12 +161,36 @@ pub enum RequestKey { CfHeaders(dashcore::BlockHash), /// `getcfilters` — keyed by the start height. CFilters(u32), - /// `getmnlistdiff` — keyed by the target block hash. + /// `getmnlistdiff` — keyed by the target block hash. Tracked for de-dup and + /// retry only; unlike the others it is not counted against a peer's in-flight + /// budget (masternode diffs are few and not throughput-paced). MnListDiff(dashcore::BlockHash), /// Block `getdata` — keyed by the requested block hash. Block(dashcore::BlockHash), } +/// Where a broker-tracked request is in its lifecycle. +enum ReqState { + /// Accepted by `send` and sitting in the `MsgQueue`, not yet on the wire. + /// Held here only for de-dup; the message itself lives in the queue. + Queued, + /// Sent to a peer and awaiting a response. Carries the message so the timeout + /// monitor can re-inject it (retry) after dropping the peer. Boxed: it holds a + /// whole `NetworkMessage`, far larger than the `Queued` variant. + OnWire(Box), +} + +/// A request currently on the wire, awaiting a response or a timeout. +struct OnWire { + /// The peer the request was routed to. + peer: SocketAddr, + /// When the request actually left the router for that peer. + sent_at: Instant, + /// The exact message, re-queued verbatim on timeout (requests are + /// self-contained, so re-sending the same bytes is a valid retry). + msg: NetworkMessage, +} + pub struct PeerNetworkManager { connected_peers: Arc>>, other_peers: Arc>>, @@ -160,6 +198,8 @@ pub struct PeerNetworkManager { msg_queue: Arc, inbound_tx: UnboundedSender, subscribers: Subscribers, + /// Broker state for every request in play (queued or on the wire). + requests: Registry, events_tx: broadcast::Sender, /// Best tip advertised by the peers, learned in `start`. Shared with the /// reconnector so its `PeersUpdated` carries the real height. @@ -231,8 +271,9 @@ impl PeerNetworkManager { let (inbound_tx, inbound_rx) = mpsc::unbounded_channel(); let subscribers: Subscribers = Arc::new(Mutex::new(HashMap::new())); - // Sized generously: `RequestOnFlight` is emitted per sent request, so the - // network-event bus is higher-volume than the peer-state events alone. + let requests: Registry = Arc::new(Mutex::new(HashMap::new())); + // Sized generously: peer-connect churn plus one `RequestTimedOut` per + // dead request during a bad-peer storm. let (events_tx, _) = broadcast::channel(4096); let shutdown = CancellationToken::new(); // Total bytes read from all peers (download-only) and the global in-flight @@ -264,6 +305,14 @@ impl PeerNetworkManager { global_cap.clone(), events_tx.clone(), best_tip.clone(), + requests.clone(), + ); + + spawn_timeout_monitor( + requests.clone(), + connected_peers.clone(), + msg_queue.clone(), + shutdown.clone(), ); spawn_bandwidth_controller( @@ -292,6 +341,7 @@ impl PeerNetworkManager { msg_queue, inbound_tx, subscribers, + requests, events_tx, best_tip, max_peers, @@ -345,7 +395,28 @@ impl PeerNetworkManager { self.shutdown.cancel(); } + /// Ask the broker to make a request. De-duplicated by request identity: if the + /// same request is already queued or on the wire, this is a no-op. Pipelines + /// exploit that to re-declare what they want each tick without tracking what + /// they already sent — the broker paces it, times it out and retries it. + /// + /// Non-pipeline messages (tx, mempool, control `getdata`) carry no request key + /// and are neither de-duplicated nor tracked; they just go on the queue. pub async fn send(&self, msg: NetworkMessage) { + let keys = request_keys(&msg); + if keys.is_empty() { + self.msg_queue.push(msg).await; + return; + } + { + let mut reqs = self.requests.lock().await; + if keys.iter().any(|k| reqs.contains_key(k)) { + return; // already in play + } + for key in keys { + reqs.insert(key, ReqState::Queued); + } + } self.msg_queue.push(msg).await; } @@ -389,6 +460,16 @@ impl PeerNetworkManager { self.msg_queue.notify.notify_one(); } + /// Report that a request's response arrived, so the broker stops tracking it + /// (no timeout, no retry, and its key is free to be requested again). The + /// owning manager calls this once it has correlated a response back to the + /// request key — the broker can't do it generically, since some responses + /// (e.g. an empty `headers`) carry nothing to match on. No-op if the key is + /// already gone (timed out first). + pub async fn request_answered(&self, key: RequestKey) { + self.requests.lock().await.remove(&key); + } + pub fn broadcast(&self, msg: NetworkMessage) { let peers = self.connected_peers.clone(); tokio::spawn(async move { @@ -441,6 +522,7 @@ fn spawn_router( global_cap: Arc, events: broadcast::Sender, best_tip: Arc, + requests: Registry, ) -> JoinHandle<()> { tokio::spawn(async move { loop { @@ -482,74 +564,30 @@ fn spawn_router( let peers = connected.lock().await; let sent = - route_tick(&queue, &peers, global_cap.load(Ordering::Relaxed), &events).await; + route_tick(&queue, &peers, global_cap.load(Ordering::Relaxed), &requests).await; drop(peers); if sent == 0 { // Queue non-empty but every peer is at its in-flight cap: wait for - // a response to free capacity (the pump notifies on each response) - // — but never wait forever, because some of those "in-flight" - // requests may be dead. A peer that stops answering keeps its - // in-flight slots raised (the pipeline times the request out and - // re-queues it, but the peer is never told), so with enough dead - // requests every peer looks full, no response is coming, and the - // router would sleep on a notify that never fires — a hard sync - // stall with a backed-up queue. On the timeout we reclaim those - // slots and drop the peers that leaked them: they have had far - // longer than the pipelines' own timeout to answer, and leaving - // them connected would just route the freed work straight back to - // the peer that ignored it (the router picks the emptiest peer). + // a response to free a slot (the pump notifies on each response) or + // for the timeout monitor to kick a stalled peer (which notifies + // too). The sleep is only a backstop against a missed wake — dead + // in-flight slots are reclaimed by the monitor dropping the peer + // that holds them, not here. tokio::select! { _ = shutdown.cancelled() => break, _ = queue.notify.notified() => {}, - _ = tokio::time::sleep(STALL_CHECK) => { - let mut peers = connected.lock().await; - let before = peers.len(); - let mut reclaimed = 0; - // Manual drain/rebuild rather than `Vec::retain`: reaping now - // awaits the per-peer latency lock, and a `retain` closure is - // synchronous. - let mut kept = Vec::with_capacity(peers.len()); - for (p, s) in peers.drain(..) { - let n = p.reap_stale(STALE_REQUEST).await; - if n == 0 { - kept.push((p, s)); - continue; - } - reclaimed += n; - // Drop the peer only if it has NEVER answered anything — - // that one is dead weight, and leaving it connected would - // route the freed work straight back to it (the router - // picks the emptiest peer). A peer that HAS served us is - // merely slow: a 2MB block over a weak link legitimately - // takes a while, and dropping those cost us 14 of 16 - // peers mid-sync. We only wanted the slot back. - let (completed, _) = p.latency_totals(); - if completed == 0 { - p.close(); - } else { - kept.push((p, s)); - } - } - *peers = kept; - if reclaimed > 0 { - tracing::warn!( - target: "dash_spv::network", - "reclaimed {} stalled request slot(s); dropped {} never-responding peer(s) -> {} connected", - reclaimed, - before - peers.len(), - peers.len(), - ); - } - } + _ = tokio::time::sleep(STALL_CHECK) => {}, } } } }) } -/// Extract the pipeline key of a router-paced request, so the router can report -/// it as on-flight. Returns `None` for non-pipeline messages. +/// Extract the pipeline keys of a router-paced request, so the router can record +/// it in the outstanding-request registry. Returns empty for non-pipeline +/// messages (mirrors `is_pipeline_request` in `peer`: only these count toward a +/// peer's in-flight and are timed out). fn request_keys(msg: &NetworkMessage) -> Vec { match msg { NetworkMessage::GetHeaders(m) | NetworkMessage::GetHeaders2(m) => { @@ -574,7 +612,7 @@ async fn route_tick( queue: &MsgQueue, peers: &[(ConnectedPeer, State)], global_cap: usize, - events: &broadcast::Sender, + requests: &Registry, ) -> usize { if peers.is_empty() { return 0; @@ -603,6 +641,9 @@ async fn route_tick( // when the router reports the request on the wire, so a dropped message is // never re-sent and never times out. let mut unsent: Vec = Vec::new(); + // Messages that made it onto the wire this round, recorded in the broker in one + // lock acquisition after the send loop. + let mut on_wire: Vec<(NetworkMessage, SocketAddr, Instant)> = Vec::new(); let mut msgs = msgs.into_iter(); for msg in msgs.by_ref() { // Send to the peer with the most free measured capacity. @@ -616,11 +657,8 @@ async fn route_tick( }; if peer.send(&msg).await.is_ok() { sent += 1; - // Tell the owning pipeline this request is now on the wire so it can - // start the response timeout from here, not from when it was queued. - for key in request_keys(&msg) { - let _ = events.send(NetworkEvent::RequestOnFlight(key)); - } + // Record which peer got it and when, so the monitor can time it out. + on_wire.push((msg, peer.addr(), Instant::now())); } else { tracing::warn!(target: "dash_spv::network", "router: send to {} failed", peer.addr()); unsent.push(msg); @@ -629,6 +667,24 @@ async fn route_tick( unsent.extend(msgs); // whatever the loop never reached queue.push_front_all(unsent).await; + if !on_wire.is_empty() { + let mut reqs = requests.lock().await; + for (msg, peer, sent_at) in on_wire { + for key in request_keys(&msg) { + // Transition Queued -> OnWire, keeping the message for retry. Skip + // keys no longer present (cancelled while queued): the request went + // out but we don't track it, so its response is simply ignored. + if let Some(slot) = reqs.get_mut(&key) { + *slot = ReqState::OnWire(Box::new(OnWire { + peer, + sent_at, + msg: msg.clone(), + })); + } + } + } + } + if sent > 0 { tracing::debug!( target: "dash_spv::network", @@ -849,6 +905,100 @@ fn spawn_reconnector( }) } +/// Time requests out and evict the peers that ignored them. +/// +/// Scans the broker for any peer sitting on an on-wire request older than +/// [`REQUEST_TIMEOUT`] and kicks it immediately: every request routed to it is now +/// dead, so we pull ALL its on-wire entries, re-inject their messages (retry to a +/// fresh peer), and drop the connection. Its in-flight slots die with it — no +/// separate reclaim — and the reconnector refills the peer set. +/// +/// Immediate kick is deliberate: a peer that has answered nothing for 30s is not +/// worth a second chance, and leaving it connected would just route the freed +/// work straight back to it (the router fills the emptiest peer first). +fn spawn_timeout_monitor( + requests: Registry, + connected: Arc>>, + queue: Arc, + shutdown: CancellationToken, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(TIMEOUT_CHECK); + loop { + tokio::select! { + _ = shutdown.cancelled() => break, + _ = ticker.tick() => {} + } + + let now = Instant::now(); + // Peers with at least one on-wire request past the timeout. + let culprits: HashSet = { + let reqs = requests.lock().await; + reqs.values() + .filter_map(|s| match s { + ReqState::OnWire(o) if now.duration_since(o.sent_at) > REQUEST_TIMEOUT => { + Some(o.peer) + } + _ => None, + }) + .collect() + }; + if culprits.is_empty() { + continue; + } + + // Pull every on-wire request routed to a culprit (fresh ones included — + // the connection is going away, so they are dead too) and re-inject its + // message (key kept as Queued so de-dup still holds). + let mut reinject: Vec = Vec::new(); + { + let mut reqs = requests.lock().await; + let keys: Vec = reqs + .iter() + .filter_map(|(k, s)| match s { + ReqState::OnWire(o) if culprits.contains(&o.peer) => Some(k.clone()), + _ => None, + }) + .collect(); + for key in keys { + if let Some(ReqState::OnWire(o)) = reqs.insert(key, ReqState::Queued) { + reinject.push(o.msg); + } + } + } + + // Drop the culprits still in the active set (some may already be gone + // — a retired-drained peer isn't here — which is fine). + let dropped = { + let mut peers = connected.lock().await; + let before = peers.len(); + peers.retain(|(p, _)| { + if culprits.contains(&p.addr()) { + p.close(); + false + } else { + true + } + }); + before - peers.len() + }; + + tracing::warn!( + target: "dash_spv::network", + "request timeout: kicked {} peer(s), retried {} request(s)", + dropped, + reinject.len(), + ); + + for msg in reinject { + queue.push(msg).await; + } + // Freed capacity (dropped peers) — wake the router to re-evaluate. + queue.notify.notify_one(); + } + }) +} + #[allow(clippy::too_many_arguments)] async fn add_peers( connected: &Mutex>, @@ -907,7 +1057,7 @@ async fn add_peers( /// occasionally stalling whole header segments for a run. Instead keep the /// connection alive in the background: its reader keeps delivering responses and /// decrementing `in_flight`. Close it once it has drained, or after -/// `STALE_REQUEST` (a peer that never drains is dead), whichever comes first. +/// `RETIRE_DRAIN_CAP` (a peer that never drains is dead), whichever comes first. fn retire_drained(peer: ConnectedPeer, shutdown: CancellationToken) { if peer.in_flight() == 0 { peer.close(); @@ -915,7 +1065,7 @@ fn retire_drained(peer: ConnectedPeer, shutdown: CancellationToken) { } tokio::spawn(async move { let mut waited = Duration::ZERO; - while peer.in_flight() > 0 && waited < STALE_REQUEST { + while peer.in_flight() > 0 && waited < RETIRE_DRAIN_CAP { tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(DRAIN_POLL) => waited += DRAIN_POLL, diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 0ba80df3b..5b92b66d0 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -21,25 +21,6 @@ impl Latency { self.pending.lock().await.push_back(Instant::now()); } - /// Drop pending sends older than `timeout` (the pipelines have long since - /// timed them out and re-queued them elsewhere), returning how many. Their - /// in-flight slots would otherwise be held forever — see - /// [`ConnectedPeer::reap_stale`]. - async fn reap(&self, timeout: Duration) -> usize { - let mut pending = self.pending.lock().await; - let now = Instant::now(); - let mut n = 0; - while let Some(&front) = pending.front() { - if now.duration_since(front) > timeout { - pending.pop_front(); - n += 1; - } else { - break; // FIFO: the rest are younger - } - } - n - } - /// Pop the oldest pending send and record its round-trip. async fn complete_one(&self) { let sent = self.pending.lock().await.pop_front(); @@ -51,13 +32,6 @@ impl Latency { } } - /// Cumulative (completed request count, total service-time nanoseconds). - /// Deltas between two reads give the windowed completion rate and average - /// service time used to size the in-flight budget by Little's Law. - fn totals(&self) -> (u64, u64) { - (self.count.load(Ordering::Relaxed), self.total_ns.load(Ordering::Relaxed)) - } - /// (completed request count, average ms, worst ms). fn snapshot(&self) -> (u64, f64, f64) { let count = self.count.load(Ordering::Relaxed); @@ -212,37 +186,11 @@ impl ConnectedPeer { } } - /// Reclaim the in-flight slots of requests this peer has not answered within - /// `timeout`, returning how many. Without this they leak: when a pipeline - /// times a request out it re-queues it, but nothing tells the peer its - /// request died, so the peer's `in_flight` counter stays raised forever. - /// Enough dead requests and EVERY peer sits at its cap with nothing actually - /// on the wire — the router sees zero free capacity, stops sending, and the - /// sync deadlocks with a full queue (reproduced at 32 peers: 0 MB/s, queue - /// 140, no sends). The count also identifies the peer as unresponsive, so the - /// caller can drop it rather than keep handing it work. - pub(crate) async fn reap_stale(&self, timeout: Duration) -> usize { - let n = self.latency.reap(timeout).await; - if n > 0 { - let _ = self - .in_flight - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| Some(v.saturating_sub(n))); - } - n - } - /// Per-peer response latency: (completed requests, average ms, worst ms). pub(crate) fn latency_stats(&self) -> (u64, f64, f64) { self.latency.snapshot() } - /// Cumulative (completed request count, total service-time ns) for this peer. - /// The bandwidth controller diffs these across a window to get the download - /// completion rate and service time that size the in-flight budget. - pub(crate) fn latency_totals(&self) -> (u64, u64) { - self.latency.totals() - } - /// Handshake round-trip latency in ms (0 if unmeasured). pub(crate) fn lag_ms(&self) -> u32 { self.lag_ms.load(Ordering::Relaxed) diff --git a/dash-spv/src/sync/block_headers/manager.rs b/dash-spv/src/sync/block_headers/manager.rs index 80c765595..c69f378bd 100644 --- a/dash-spv/src/sync/block_headers/manager.rs +++ b/dash-spv/src/sync/block_headers/manager.rs @@ -138,7 +138,7 @@ impl BlockHeadersManager { // cost (~70ms per 8000-header batch), and it otherwise runs serially on this // single manager task. Split into 4 chunks across tokio's runtime workers, then // feed the pre-hashed headers to the pipeline (validates checkpoint match). - let matched = if headers.is_empty() { + let (matched, answered) = if headers.is_empty() { self.pipeline.receive_headers_prehashed(&[])? } else { let len = headers.len(); @@ -164,6 +164,13 @@ impl BlockHeadersManager { self.pipeline.receive_headers_prehashed(&hashed)? }; + // Correlated to an outstanding GetHeaders: stop the network manager + // tracking it for timeout. Empty responses answer a request too (the + // pipeline reports which tip), so this can't be derived from `headers`. + if let Some(locator) = answered { + network.request_answered(crate::network::RequestKey::Headers(locator)).await; + } + if matched.is_none() && !headers.is_empty() { tracing::debug!( "Headers not matched by pipeline (prev_hash: {}), may be post-sync update", diff --git a/dash-spv/src/sync/block_headers/pipeline.rs b/dash-spv/src/sync/block_headers/pipeline.rs index 67ea51968..6eb8da082 100644 --- a/dash-spv/src/sync/block_headers/pipeline.rs +++ b/dash-spv/src/sync/block_headers/pipeline.rs @@ -14,6 +14,12 @@ use crate::network::PeerNetworkManager; use crate::sync::block_headers::segment_state::SegmentState; use crate::types::HashedBlockHeader; +/// Result of routing a headers batch through the pipeline: the in-memory batch a +/// segment validated (segment_id, start, end, stop_hash) if the chain advanced, +/// plus the locator hash of the request this response answered (for the network +/// manager's timeout tracking). +type PipelineReceive = (Option<(usize, u32, u32, BlockHash)>, Option); + /// Pipeline for parallel header downloads across checkpoint-defined segments. /// /// Divides the blockchain into segments based on checkpoints and downloads @@ -116,22 +122,6 @@ impl HeadersPipeline { self.segments.len() } - /// Clear timed-out in-flight requests across all segments so `send_pending` - /// re-issues them (recovers from a lost request / dead peer). - pub fn handle_timeouts(&mut self, timeout: std::time::Duration) { - for segment in &mut self.segments { - segment.handle_timeouts(timeout); - } - } - - /// Start the response timeout for the segment whose GetHeaders (locator = - /// `hash`) the router just put on the wire. No-op on the other segments. - pub fn mark_on_flight(&mut self, hash: &BlockHash) { - for segment in &mut self.segments { - segment.coordinator.mark_on_flight(&[*hash]); - } - } - /// Send pending requests for active segments. /// Returns the number of requests sent. pub async fn send_pending(&mut self, network: &Arc) -> SyncResult { @@ -151,10 +141,14 @@ impl HeadersPipeline { /// Match an already-hashed batch to the correct segment and route it. The X11 /// block hashing is done in parallel at the manager (on the tokio runtime). + /// + /// The second tuple element is the locator hash of the request this response + /// answered (if any), so the caller can tell the network manager to stop + /// tracking it for timeout. pub fn receive_headers_prehashed( &mut self, hashed: &[HashedBlockHeader], - ) -> SyncResult> { + ) -> SyncResult { if hashed.is_empty() { // Empty response means the peer has no more headers after our locator. // Route to the tip segment (target_height is None) if it has in-flight requests. @@ -162,19 +156,19 @@ impl HeadersPipeline { for segment in &mut self.segments { if !segment.complete && segment.target_height.is_none() - && segment.coordinator.active_count() > 0 + && segment.in_flight_tip.is_some() { tracing::debug!( "Routing empty response to tip segment {} at height {}", segment.segment_id, segment.current_height ); - segment.receive_headers_prehashed(hashed)?; + let (_, answered) = segment.receive_headers_prehashed(hashed)?; // Empty tip response has no batch to announce. - return Ok(None); + return Ok((None, answered)); } } - return Ok(None); + return Ok((None, None)); } let prev_hash = hashed[0].header().prev_blockhash; @@ -193,16 +187,19 @@ impl HeadersPipeline { if segment.complete && segment.target_height.is_none() { segment.complete = false; self.next_to_store = idx; - // Mark as in-flight so the coordinator accepts these unsolicited headers - segment.coordinator.mark_sent(&[prev_hash]); + // Mark as in-flight so the segment accepts these unsolicited headers + segment.in_flight_tip = Some(prev_hash); tracing::debug!( "Tip segment {} receiving post-sync headers, reset for continued processing", segment.segment_id ); } let sid = segment.segment_id; - let range = segment.receive_headers_prehashed(hashed)?; - return Ok(range.map(|(start, end, stop_hash)| (sid, start, end, stop_hash))); + let (range, answered) = segment.receive_headers_prehashed(hashed)?; + return Ok(( + range.map(|(start, end, stop_hash)| (sid, start, end, stop_hash)), + answered, + )); } } @@ -211,7 +208,7 @@ impl HeadersPipeline { let first_hash = *hashed[0].hash(); if self.segments.iter().any(|s| s.current_tip_hash == first_hash) { tracing::debug!("Ignoring duplicate header {} from another peer", first_hash); - return Ok(None); + return Ok((None, None)); } tracing::warn!( @@ -219,7 +216,7 @@ impl HeadersPipeline { hashed.len(), prev_hash ); - Ok(None) + Ok((None, None)) } /// Get segments that are ready to store (complete and in order). diff --git a/dash-spv/src/sync/block_headers/segment_state.rs b/dash-spv/src/sync/block_headers/segment_state.rs index 7a6cbfbfc..844d1a812 100644 --- a/dash-spv/src/sync/block_headers/segment_state.rs +++ b/dash-spv/src/sync/block_headers/segment_state.rs @@ -1,6 +1,5 @@ use crate::error::{SyncError, SyncResult}; use crate::network::PeerNetworkManager; -use crate::sync::download_coordinator::DownloadCoordinator; use crate::types::HashedBlockHeader; use dashcore::hashes::Hash; use dashcore::network::message::NetworkMessage; @@ -8,6 +7,12 @@ use dashcore::network::message_blockdata::GetHeadersMessage; use dashcore::BlockHash; use std::sync::Arc; +/// Result of routing a headers batch to a segment: the in-memory batch it +/// validated (start, end, stop_hash) if the chain advanced, plus the locator +/// hash of the request this response answered (for the network manager's +/// timeout tracking). +type SegmentReceive = (Option<(u32, u32, BlockHash)>, Option); + /// State for a single download segment between two checkpoints. #[derive(Debug)] pub(super) struct SegmentState { @@ -23,8 +28,10 @@ pub(super) struct SegmentState { pub(super) current_tip_hash: BlockHash, /// Current height reached in this segment. pub(super) current_height: u32, - /// Download coordinator for tracking in-flight requests. - pub(super) coordinator: DownloadCoordinator, + /// The locator hash of the one in-flight `getheaders`, if any. The protocol is + /// sequential (one request per segment at a time), so a single slot suffices; + /// the network manager owns its timeout and retries by re-injecting. + pub(super) in_flight_tip: Option, /// Buffered headers waiting to be stored. pub(super) buffered_headers: Vec, /// Whether this segment has completed downloading. @@ -47,7 +54,7 @@ impl SegmentState { target_hash, current_tip_hash: start_hash, current_height: start_height, - coordinator: DownloadCoordinator::new(), + in_flight_tip: None, buffered_headers: Vec::new(), complete: false, } @@ -56,22 +63,7 @@ impl SegmentState { /// Check if the segment can send more requests. /// Only one getheaders request can be in-flight at a time (sequential protocol). pub(super) fn can_send(&self) -> bool { - !self.complete && !self.coordinator.is_in_flight(&self.current_tip_hash) - } - - /// Drop in-flight requests that timed out. Removing the tip hash from the - /// in-flight set makes `can_send` true again, so the next `send_pending` - /// re-issues the GetHeaders to (likely) a different peer. - pub(super) fn handle_timeouts(&mut self, timeout: std::time::Duration) { - let timed_out = self.coordinator.take_timed_out(timeout); - if !timed_out.is_empty() { - tracing::warn!( - "Segment {}: {} request(s) timed out at height {}, will re-send", - self.segment_id, - timed_out.len(), - self.current_height - ); - } + !self.complete && self.in_flight_tip.is_none() } /// Send a GetHeaders request for this segment. @@ -85,7 +77,7 @@ impl SegmentState { BlockHash::all_zeros(), ))) .await; - self.coordinator.mark_sent(&[self.current_tip_hash]); + self.in_flight_tip = Some(self.current_tip_hash); tracing::debug!( "Segment {}: sent GetHeaders from height {} hash {}", self.segment_id, @@ -105,21 +97,28 @@ impl SegmentState { /// Route an already-hashed batch to this segment. The expensive X11 block /// hashing is done in parallel off this task (at the manager, on the tokio /// runtime), so here we only run the cheap sequential chain/checkpoint check. + /// + /// Also returns the locator hash of the request this response answered (if it + /// matched an outstanding request), so the caller can tell the network manager + /// to stop tracking it for timeout. An empty response still answers a request + /// (the current tip's), which is why the caller can't reconstruct the key from + /// the response bytes alone. pub(super) fn receive_headers_prehashed( &mut self, hashed: &[HashedBlockHeader], - ) -> SyncResult> { + ) -> SyncResult { if hashed.is_empty() { // Empty response means we've reached the peer's tip for this segment self.complete = true; // Clear in-flight tracking for the current tip hash - self.coordinator.receive(&self.current_tip_hash); + let answered = self.current_tip_hash; + self.in_flight_tip = None; tracing::info!( "Segment {}: complete (empty response at height {})", self.segment_id, self.current_height ); - return Ok(None); + return Ok((None, Some(answered))); } // Reject headers on a segment that already reached its checkpoint @@ -134,12 +133,13 @@ impl SegmentState { // Mark the request as received, reject if we never requested this hash let prev_hash = hashed[0].header().prev_blockhash; - if !self.coordinator.receive(&prev_hash) { + if self.in_flight_tip != Some(prev_hash) { return Err(SyncError::InvalidState(format!( "Segment {}: received unrequested headers (prev_hash {})", self.segment_id, prev_hash ))); } + self.in_flight_tip = None; // Sequentially validate the chain link to the target checkpoint and // buffer, using the precomputed hashes. @@ -200,11 +200,13 @@ impl SegmentState { self.buffered_headers.len() ); + // `prev_hash` matched an outstanding request (receive above succeeded), so + // it is the answered locator key regardless of how many headers advanced. if processed > 0 { // (start, end, stop_hash) of the batch just validated in memory. - Ok(Some((start_height, self.current_height, last_hash))) + Ok((Some((start_height, self.current_height, last_hash)), Some(prev_hash))) } else { - Ok(None) + Ok((None, Some(prev_hash))) } } @@ -219,6 +221,6 @@ impl SegmentState { /// are preserved so a reconnect can resume from where the last peer left off /// without re-fetching headers we already have. pub(super) fn clear_in_flight(&mut self) { - self.coordinator.clear(); + self.in_flight_tip = None; } } diff --git a/dash-spv/src/sync/block_headers/sync_manager.rs b/dash-spv/src/sync/block_headers/sync_manager.rs index e37b7e7f7..b76bc80e9 100644 --- a/dash-spv/src/sync/block_headers/sync_manager.rs +++ b/dash-spv/src/sync/block_headers/sync_manager.rs @@ -16,11 +16,6 @@ use std::time::{Duration, Instant}; /// Timeout waiting for unsolicited header messages after a block announcement. pub(super) const UNSOLICITED_HEADERS_WAIT_TIMEOUT: Duration = Duration::from_secs(3); -/// A GetHeaders with no response after this long is assumed lost (the peer died -/// or dropped it) and re-sent. Segments are chained, so one lost request stalls -/// the whole sequential store head until it's retried. -pub(super) const HEADER_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); - #[async_trait] impl SyncManager for BlockHeadersManager { fn identifier(&self) -> ManagerIdentifier { @@ -44,12 +39,6 @@ impl SyncManager for BlockHeadersMana &[MessageType::Headers, MessageType::Inv] } - fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { - if let crate::network::RequestKey::Headers(hash) = key { - self.pipeline.mark_on_flight(hash); - } - } - fn on_disconnect(&mut self) { // Drop only per-peer in-flight bookkeeping. Segment topology and // validated chain state per segment (current_tip_hash, current_height, @@ -147,9 +136,9 @@ impl SyncManager for BlockHeadersMana return Ok(vec![]); } - // During initial sync, re-send any timed-out requests then push pending. + // During initial sync, push pending. Timeouts/retry are the network + // manager's job now (it re-queues via `requeue_timed_out`). if self.state() == SyncState::Syncing { - self.pipeline.handle_timeouts(HEADER_REQUEST_TIMEOUT); let sent = self.pipeline.send_pending(network).await?; if sent > 0 { tracing::debug!("Tick: pipeline sent {} more requests", sent); diff --git a/dash-spv/src/sync/blocks/pipeline.rs b/dash-spv/src/sync/blocks/pipeline.rs index 9f154fa73..f7be7d6d5 100644 --- a/dash-spv/src/sync/blocks/pipeline.rs +++ b/dash-spv/src/sync/blocks/pipeline.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use crate::error::SyncResult; use crate::network::PeerNetworkManager; -use crate::sync::download_coordinator::DownloadCoordinator; use crate::types::HashedBlock; use dashcore::network::message::NetworkMessage; use dashcore::network::message_blockdata::Inventory; @@ -16,16 +15,17 @@ use key_wallet_manager::{FilterMatchKey, WalletId}; /// Pipeline for downloading blocks with height-ordered processing. /// -/// This is a thin wrapper that handles building GetData inventory messages. -/// Tracks block heights to enable ordered processing and buffers downloaded blocks. +/// Holds no request queue of its own: it declares the blocks it wants to the +/// network manager (the broker de-duplicates, paces, times out and retries), and +/// buffers the arrivals for height-ordered processing. A block is "wanted" for +/// exactly as long as it sits in `hash_to_height`. pub(super) struct BlocksPipeline { - /// Coordinates pending/in-flight block hashes for download. - coordinator: DownloadCoordinator, - /// Heights queued or in-flight (waiting for download). + /// Heights still wanted (block requested, not yet downloaded). pending_heights: BTreeSet, /// Downloaded blocks ready to process (height -> block, with its cached hash). downloaded: BTreeMap, - /// Map hash -> height for looking up height when block arrives. + /// Wanted blocks: hash -> height. A block leaves this map once downloaded. + /// Doubles as the "is this block wanted?" set for validating arrivals. hash_to_height: HashMap, /// Per-block interested wallets, populated when the block is queued. /// Only those wallets get the block processed. @@ -35,9 +35,9 @@ pub(super) struct BlocksPipeline { impl std::fmt::Debug for BlocksPipeline { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("BlocksPipeline") - .field("coordinator", &self.coordinator) .field("pending_heights", &self.pending_heights.len()) .field("downloaded", &self.downloaded.len()) + .field("wanted", &self.hash_to_height.len()) .finish() } } @@ -52,7 +52,6 @@ impl BlocksPipeline { /// Create a new blocks pipeline. pub(super) fn new() -> Self { Self { - coordinator: DownloadCoordinator::new(), pending_heights: BTreeSet::new(), downloaded: BTreeMap::new(), hash_to_height: HashMap::new(), @@ -70,7 +69,6 @@ impl BlocksPipeline { let already_tracked = self.hash_to_height.contains_key(&hash) || self.hash_to_wallets.contains_key(&hash); if !already_tracked { - self.coordinator.enqueue([hash]); self.pending_heights.insert(key.height()); self.hash_to_height.insert(hash, key.height()); } @@ -80,86 +78,56 @@ impl BlocksPipeline { /// Check if the pipeline has completed all work. /// - /// Returns true when no blocks are pending, downloading, or waiting to be processed. + /// Returns true when no blocks are wanted, downloading, or waiting to be processed. pub(super) fn is_complete(&self) -> bool { - self.coordinator.is_empty() && self.downloaded.is_empty() && self.pending_heights.is_empty() + self.hash_to_height.is_empty() + && self.downloaded.is_empty() + && self.pending_heights.is_empty() } - /// Check if there are pending requests to make. + /// Check if there are blocks still to download. pub(super) fn has_pending_requests(&self) -> bool { - self.coordinator.available_to_send() > 0 + !self.hash_to_height.is_empty() } - /// Start the response timeout for a block the router just put on the wire. - pub(super) fn mark_on_flight(&mut self, hash: &BlockHash) { - self.coordinator.mark_on_flight(std::slice::from_ref(hash)); - } - - /// Re-queue block downloads whose response never arrived (peer died or the - /// block was lost), so the next `send_pending` re-issues the GetData. - pub(super) fn handle_timeouts(&mut self, timeout: std::time::Duration) { - let timed_out = self.coordinator.take_timed_out(timeout); - if !timed_out.is_empty() { - tracing::warn!("Blocks: re-queuing {} timed-out block(s)", timed_out.len()); - self.coordinator.enqueue(timed_out); - } - } - - /// Offer every pending block to the network manager. + /// Declare every wanted block to the network manager. /// - /// Blocks go through the router like every other request — one `getdata` per - /// block, so a request is one in-flight unit on the peer and one `block` in - /// reply, and the peer's measured capacity throttles them. (They used to bypass - /// the router entirely, a direct-send from the days when the single FIFO left - /// them stuck behind hundreds of `getcfilters`. The class-based queue fixed that - /// starvation properly; unpaced, the bypass then buried peers under thousands of - /// 2MB requests, which they silently dropped, hanging the sync on blocks that - /// were never served.) + /// Fired freely (on queue, on arrival, on tick): the broker de-duplicates, so + /// re-declaring a block already queued or on the wire is a no-op, and it owns + /// pacing (one `getdata` per block, throttled by each peer's measured capacity) + /// and retry (re-inject on timeout after dropping the dead peer). Re-declaring + /// each tick is the safety net if a peer drops before the broker retries. /// - /// Returns the number of blocks requested. + /// Returns the number of blocks declared (offered, not necessarily newly sent). pub(super) async fn send_pending( &mut self, network: &Arc, ) -> SyncResult { - let hashes = self.coordinator.take_pending(self.coordinator.available_to_send()); - if hashes.is_empty() { + if self.hash_to_height.is_empty() { return Ok(0); } - + let hashes: Vec = self.hash_to_height.keys().copied().collect(); for hash in &hashes { network.send(NetworkMessage::GetData(vec![Inventory::Block(*hash)])).await; } - // Handed to the network; the response timeout starts when the router reports - // each one actually on the wire (`RequestKey::Block` -> `mark_on_flight`). - self.coordinator.mark_sent(&hashes); - - tracing::debug!( - "Requested {} blocks ({} in flight, {} pending)", - hashes.len(), - self.coordinator.active_count(), - self.coordinator.pending_count() - ); - + tracing::debug!("Declared {} wanted block(s) to the broker", hashes.len()); Ok(hashes.len()) } /// Handle a received block using internal height mapping. /// /// Stores the block in the downloaded buffer for height-ordered processing and - /// returns **the height it was queued at**, or `None` if the block was never - /// requested. `queue` records the height alongside the hash, so the caller gets it - /// from here instead of asking the storage's chain-wide hash index to recover what - /// this pipeline already knew. + /// returns **the height it was queued at**, or `None` if the block was not + /// wanted (unrequested, or already received). `queue` records the height + /// alongside the hash, so the caller gets it from here instead of asking the + /// storage's chain-wide hash index to recover what this pipeline already knew. pub(super) fn receive_block(&mut self, block: &HashedBlock) -> Option { let hash = *block.hash(); - if !self.coordinator.receive(&hash) { + // Not in the wanted set => unrequested or already downloaded; ignore. + let Some(height) = self.hash_to_height.remove(&hash) else { tracing::debug!("Ignoring unrequested block: {}", hash); return None; - } - - // `queue` populates the coordinator and `hash_to_height` together, so a block - // the coordinator accepted always has a height here. - let height = self.hash_to_height.remove(&hash)?; + }; self.pending_heights.remove(&height); self.downloaded.insert(height, block.clone()); diff --git a/dash-spv/src/sync/blocks/sync_manager.rs b/dash-spv/src/sync/blocks/sync_manager.rs index 8030328b9..4e10060cc 100644 --- a/dash-spv/src/sync/blocks/sync_manager.rs +++ b/dash-spv/src/sync/blocks/sync_manager.rs @@ -32,14 +32,6 @@ impl SyncManager for BlocksManage &[crate::network::MessageType::Block] } - /// The router put a block request on the wire: start ITS response timeout from - /// here, not from when it was queued — the queue can hold it for a while. - fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { - if let crate::network::RequestKey::Block(hash) = key { - self.pipeline.mark_on_flight(hash); - } - } - async fn start_sync( &mut self, _network: &Arc, @@ -90,6 +82,10 @@ impl SyncManager for BlocksManage return Ok(vec![]); }; + // Response correlated: stop the network manager tracking this request for + // timeout. + network.request_answered(crate::network::RequestKey::Block(*hashed_block.hash())).await; + tracing::debug!("Received block {} at height {}", hashed_block.hash(), height); // Persist blocks to speed-up wallet rescans @@ -100,9 +96,10 @@ impl SyncManager for BlocksManage // Process buffered blocks let events = self.process_buffered_blocks().await?; - if self.pipeline.has_pending_requests() { - self.send_pending(network).await?; - } + // No `send_pending` here: the wanted blocks are already declared to the + // broker, which paces them out as capacity frees. Re-declaring per received + // block would re-scan the whole wanted set on the hot path. New work is + // declared on `BlocksNeeded` and topped up on tick. Ok(events) } @@ -190,8 +187,8 @@ impl SyncManager for BlocksManage } async fn tick(&mut self, network: &Arc) -> SyncResult> { - // Re-issue any timed-out block downloads, then send pending. - self.pipeline.handle_timeouts(std::time::Duration::from_secs(30)); + // Timeouts/retry are the network manager's job now (it re-queues via + // `requeue_timed_out`); just push whatever is pending. self.send_pending(network).await?; // Try to process any buffered blocks diff --git a/dash-spv/src/sync/download_coordinator.rs b/dash-spv/src/sync/download_coordinator.rs deleted file mode 100644 index 69df27af4..000000000 --- a/dash-spv/src/sync/download_coordinator.rs +++ /dev/null @@ -1,239 +0,0 @@ -//! Generic download coordinator for pipelined downloads. -//! -//! A pending queue plus an in-flight map (item -> send time). Pacing is the -//! network manager's job, but retry lives here: [`take_timed_out`] surfaces -//! requests that were sent but never answered (a peer died, or the response was -//! lost) so the pipeline can re-issue them. -//! -//! [`take_timed_out`]: DownloadCoordinator::take_timed_out - -use std::collections::{HashMap, VecDeque}; -use std::hash::Hash; -use std::time::{Duration, Instant}; - -/// Generic download coordinator: a pending queue plus an in-flight map. -/// -/// Generic over the key type `K` which identifies download items. -/// Use `u32` for height-based downloads, `BlockHash` for hash-based. -#[derive(Debug)] -pub(crate) struct DownloadCoordinator { - /// Items waiting to be requested. - pending: VecDeque, - /// Items handed to the network, mapped to their on-the-wire time. `None` = - /// still queued in the router (not yet sent); `Some(t)` = actually sent to a - /// peer at `t` (set from the network manager's `RequestOnFlight`). The - /// response timeout counts from `Some(t)`; queued items never time out, since - /// the queue is drained in order and their send — hence `RequestOnFlight` — - /// is guaranteed to come. - in_flight: HashMap>, -} - -impl Default for DownloadCoordinator { - fn default() -> Self { - Self { - pending: VecDeque::new(), - in_flight: HashMap::new(), - } - } -} - -impl DownloadCoordinator { - /// Create an empty coordinator. - pub(crate) fn new() -> Self { - Self::default() - } - - /// Clear all state. - pub(crate) fn clear(&mut self) { - self.pending.clear(); - self.in_flight.clear(); - } - - /// Queue items for download. - pub(crate) fn enqueue(&mut self, items: impl IntoIterator) { - for item in items { - self.pending.push_back(item); - } - } - - /// Get the number of items available to send. - /// - /// No concurrency cap: the network manager coordinates pacing, so - /// everything pending is offered. - pub(crate) fn available_to_send(&self) -> usize { - self.pending.len() - } - - /// Take items from the pending queue (up to count). - /// - /// Items are removed from pending but NOT yet marked as in-flight. - /// Call `mark_sent` after successfully sending the request. - pub(crate) fn take_pending(&mut self, count: usize) -> Vec { - let actual = count.min(self.pending.len()); - let mut items = Vec::with_capacity(actual); - for _ in 0..actual { - if let Some(item) = self.pending.pop_front() { - items.push(item); - } - } - items - } - - /// Mark items as handed to the network (queued). The response timeout does - /// NOT start yet — it starts when `mark_on_flight` reports the request was - /// actually sent to a peer. - pub(crate) fn mark_sent(&mut self, items: &[K]) { - for item in items { - self.in_flight.insert(item.clone(), None); - } - } - - /// Mark items as actually ON THE WIRE (sent to a peer), starting their - /// response timeout. Driven by the network manager's `RequestOnFlight` event. - /// No-op for items already answered/removed or already on-flight. - pub(crate) fn mark_on_flight(&mut self, items: &[K]) { - let now = Instant::now(); - for item in items { - if let Some(slot @ None) = self.in_flight.get_mut(item) { - *slot = Some(now); - } - } - } - - /// Handle a received item. - /// - /// Returns true if the item was being tracked, false if unexpected. - pub(crate) fn receive(&mut self, key: &K) -> bool { - self.in_flight.remove(key).is_some() - } - - /// Remove and return every in-flight item that has been outstanding longer - /// than `timeout` (a peer died or the response was lost). The items leave the - /// in-flight set, so the pipeline simply re-sends them next tick — segments - /// via `can_send`, pending-driven pipelines by re-`enqueue`ing what's returned. - pub(crate) fn take_timed_out(&mut self, timeout: Duration) -> Vec { - let now = Instant::now(); - // Only requests that are actually ON THE WIRE can time out. A request - // still sitting in the router queue (`on_flight` = None) is NEVER timed - // out here: the queue drains in order and items are not dropped, so its - // `RequestOnFlight` is guaranteed to fire eventually — timing it out from - // queue time would re-issue a request that is merely waiting its turn - // behind a deep queue, a spurious duplicate. - let timed_out: Vec = self - .in_flight - .iter() - .filter(|(_, on_flight)| { - on_flight.is_some_and(|sent| now.duration_since(sent) > timeout) - }) - .map(|(k, _)| k.clone()) - .collect(); - for k in &timed_out { - self.in_flight.remove(k); - } - timed_out - } - - /// Drop a key from the pending queue without touching in-flight state. - /// - /// Used when a pending item is satisfied through a side channel (e.g. a - /// batch completed via a different path) so the next `take_pending` doesn't - /// resurrect a finished item. - pub(crate) fn cancel_pending(&mut self, key: &K) { - self.pending.retain(|k| k != key); - } - - /// Check if an item is currently in-flight. - pub(crate) fn is_in_flight(&self, key: &K) -> bool { - self.in_flight.contains_key(key) - } - - /// Check if the coordinator has no work (empty pending and in-flight). - pub(crate) fn is_empty(&self) -> bool { - self.pending.is_empty() && self.in_flight.is_empty() - } - - /// Get the number of pending items. - pub(crate) fn pending_count(&self) -> usize { - self.pending.len() - } - - /// Get the number of in-flight items. - pub(crate) fn active_count(&self) -> usize { - self.in_flight.len() - } - - /// Get the total remaining items (pending + in-flight). - pub(crate) fn remaining(&self) -> usize { - self.pending.len() + self.in_flight.len() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_new_coordinator() { - let coord: DownloadCoordinator = DownloadCoordinator::new(); - assert!(coord.is_empty()); - assert_eq!(coord.pending_count(), 0); - assert_eq!(coord.active_count(), 0); - } - - #[test] - fn test_enqueue_and_available() { - let mut coord: DownloadCoordinator = DownloadCoordinator::new(); - coord.enqueue([1, 2, 3, 4, 5]); - assert_eq!(coord.pending_count(), 5); - // No concurrency cap: everything pending is available. - assert_eq!(coord.available_to_send(), 5); - } - - #[test] - fn test_take_pending() { - let mut coord: DownloadCoordinator = DownloadCoordinator::new(); - coord.enqueue([1, 2, 3, 4, 5]); - - let items = coord.take_pending(3); - assert_eq!(items, vec![1, 2, 3]); - assert_eq!(coord.pending_count(), 2); - } - - #[test] - fn test_mark_sent_and_receive() { - let mut coord: DownloadCoordinator = DownloadCoordinator::new(); - coord.enqueue([1, 2, 3]); - - let items = coord.take_pending(2); - coord.mark_sent(&items); - assert_eq!(coord.active_count(), 2); - assert!(coord.is_in_flight(&1)); - assert!(coord.is_in_flight(&2)); - assert!(!coord.is_in_flight(&3)); - - assert!(coord.receive(&1)); - assert!(!coord.is_in_flight(&1)); - // Receiving an untracked key returns false. - assert!(!coord.receive(&99)); - } - - #[test] - fn test_cancel_pending() { - let mut coord: DownloadCoordinator = DownloadCoordinator::new(); - coord.enqueue([1, 2, 3]); - coord.cancel_pending(&2); - assert_eq!(coord.take_pending(3), vec![1, 3]); - } - - #[test] - fn test_clear_and_remaining() { - let mut coord: DownloadCoordinator = DownloadCoordinator::new(); - coord.enqueue([1, 2, 3]); - coord.mark_sent(&[4]); - assert_eq!(coord.remaining(), 4); - - coord.clear(); - assert!(coord.is_empty()); - assert_eq!(coord.remaining(), 0); - } -} diff --git a/dash-spv/src/sync/filter_headers/pipeline.rs b/dash-spv/src/sync/filter_headers/pipeline.rs index 4d36780e2..98b258756 100644 --- a/dash-spv/src/sync/filter_headers/pipeline.rs +++ b/dash-spv/src/sync/filter_headers/pipeline.rs @@ -12,20 +12,20 @@ use std::sync::Arc; use crate::error::{SyncError, SyncResult}; use crate::network::PeerNetworkManager; use crate::storage::BlockHeaderStorage; -use crate::sync::download_coordinator::DownloadCoordinator; /// Batch size for filter header requests. const FILTER_HEADERS_BATCH_SIZE: u32 = 2000; /// Pipeline for downloading compact block filter headers. /// -/// Tracks batches by stop_hash using simple pending/in-flight sets, with a -/// HashMap buffer for out-of-order responses that need sequential processing. +/// Holds no request queue of its own: it declares the batches it wants to the +/// network manager (the broker de-duplicates, paces, times out and retries) and +/// stitches responses in order. A batch is "wanted" for exactly as long as it +/// sits in `batch_starts` (removed once its response arrives). #[derive(Debug)] pub(super) struct FilterHeadersPipeline { - /// Coordinator managing pending/in-flight batches (keyed by stop_hash). - coordinator: DownloadCoordinator, - /// Maps stop_hash -> start_height for each batch. + /// Wanted batches: stop_hash -> start_height. Doubles as the "is this batch + /// wanted?" set for validating arrivals. batch_starts: HashMap, /// Out-of-order response buffer (start_height -> data). buffered: HashMap, @@ -45,7 +45,6 @@ impl FilterHeadersPipeline { /// Create a new CFHeaders pipeline. pub(super) fn new() -> Self { Self { - coordinator: DownloadCoordinator::new(), batch_starts: HashMap::new(), buffered: HashMap::new(), next_expected: 0, @@ -81,7 +80,6 @@ impl FilterHeadersPipeline { SyncError::Storage(format!("Missing header at height {}", batch_end)) })?; - self.coordinator.enqueue([stop_hash]); self.batch_starts.insert(stop_hash, current); added += 1; @@ -109,7 +107,6 @@ impl FilterHeadersPipeline { /// height (== `next_expected`); responses arriving out of order are buffered /// and stitched in order exactly as in the storage path. pub(super) fn init_in_memory(&mut self, start_height: u32, target_height: u32) { - self.coordinator.clear(); self.batch_starts.clear(); self.buffered.clear(); self.next_expected = start_height; @@ -132,7 +129,6 @@ impl FilterHeadersPipeline { if end_height > self.target_height { self.target_height = end_height; } - self.coordinator.enqueue([stop_hash]); self.batch_starts.insert(stop_hash, start_height); } @@ -141,7 +137,7 @@ impl FilterHeadersPipeline { /// — i.e. a `BlockHeadersInMemory` event was dropped (lagged bus) and the /// batch at `next_expected` must be recovered from storage. pub(super) fn has_gap_at_next_expected(&self) -> bool { - self.coordinator.is_empty() + self.batch_starts.is_empty() && !self.buffered.contains_key(&self.next_expected) && self.target_height > 0 && self.next_expected <= self.target_height @@ -158,7 +154,6 @@ impl FilterHeadersPipeline { let batch_end = (start + FILTER_HEADERS_BATCH_SIZE - 1).min(self.target_height); if let Some(stop_hash) = storage.get_header(batch_end).await?.map(|h| *h.hash()) { self.batch_starts.entry(stop_hash).or_insert(start); - self.coordinator.enqueue([stop_hash]); tracing::warn!( "FilterHeaders: reconciled dropped batch [{}..{}] from storage", start, @@ -181,7 +176,7 @@ impl FilterHeadersPipeline { /// Check if the pipeline is complete. pub(super) fn is_complete(&self) -> bool { - self.coordinator.is_empty() + self.batch_starts.is_empty() && self.buffered.is_empty() && (self.target_height == 0 || self.next_expected > self.target_height) } @@ -193,13 +188,12 @@ impl FilterHeadersPipeline { start_height: u32, target_height: u32, ) -> SyncResult<()> { - self.coordinator.clear(); self.batch_starts.clear(); self.buffered.clear(); self.next_expected = start_height; self.target_height = target_height; - // Build request queue + // Build the wanted set let mut current = start_height; while current <= target_height { let batch_end = (current + FILTER_HEADERS_BATCH_SIZE - 1).min(target_height); @@ -210,15 +204,14 @@ impl FilterHeadersPipeline { SyncError::Storage(format!("Missing header at height {}", batch_end)) })?; - self.coordinator.enqueue([stop_hash]); self.batch_starts.insert(stop_hash, current); current = batch_end + 1; } tracing::info!( - "Built CFHeaders request queue: {} batches for heights {} to {}", - self.coordinator.pending_count(), + "Built CFHeaders wanted set: {} batches for heights {} to {}", + self.batch_starts.len(), start_height, target_height ); @@ -226,42 +219,30 @@ impl FilterHeadersPipeline { Ok(()) } - /// Start the response timeout for a batch the router just put on the wire. - pub(super) fn mark_on_flight(&mut self, stop_hash: &BlockHash) { - self.coordinator.mark_on_flight(&[*stop_hash]); - } - - /// Send pending requests using the network manager (synchronous). - /// Re-queue GetCFHeaders batches whose response never arrived (peer died or - /// the message was lost), so the next `send_pending` re-issues them. - pub(super) fn handle_timeouts(&mut self, timeout: std::time::Duration) { - let timed_out = self.coordinator.take_timed_out(timeout); - if !timed_out.is_empty() { - tracing::warn!("FilterHeaders: re-queuing {} timed-out batch(es)", timed_out.len()); - self.coordinator.enqueue(timed_out); - } - } - + /// Declare every wanted batch to the network manager. + /// + /// Fired freely (on new batches, on arrival, on tick): the broker + /// de-duplicates, paces and retries, so re-declaring what's already in flight + /// is a no-op and re-declaring each tick is the safety net if a peer drops. + /// + /// Returns the number of batches declared (offered, not necessarily newly sent). pub(super) async fn send_pending( &mut self, network: &Arc, ) -> SyncResult { - let count = self.coordinator.available_to_send(); - if count == 0 { + if self.batch_starts.is_empty() { return Ok(0); } - - let stop_hashes = self.coordinator.take_pending(count); - let mut sent = 0; - - for stop_hash in stop_hashes { - let Some(&start_height) = self.batch_starts.get(&stop_hash) else { - return Err(SyncError::InvalidState(format!( - "No batch_starts entry for pending stop_hash {}", - stop_hash - ))); - }; - + // Declare in ascending start_height order: `batch_starts` is a HashMap, so + // iterating it raw would fan requests out in a random order that shifts + // run-to-run. Lowest-first keeps the sequential in-order stitch fed and the + // behaviour deterministic. The broker de-duplicates, so re-declaring one + // already in flight is a no-op. + let mut batches: Vec<(u32, BlockHash)> = + self.batch_starts.iter().map(|(stop, &start)| (start, *stop)).collect(); + batches.sort_unstable_by_key(|(start, _)| *start); + let n = batches.len(); + for (start_height, stop_hash) in batches { network .send(NetworkMessage::GetCFHeaders(GetCFHeaders { filter_type: 0u8, @@ -269,21 +250,9 @@ impl FilterHeadersPipeline { stop_hash, })) .await; - - self.coordinator.mark_sent(&[stop_hash]); - - tracing::debug!( - "Sent GetCFHeaders: start={}, stop={} ({} active, {} pending)", - start_height, - stop_hash, - self.coordinator.active_count(), - self.coordinator.pending_count() - ); - - sent += 1; } - - Ok(sent) + tracing::debug!("Declared {} wanted CFHeaders batch(es) to the broker", n); + Ok(n) } /// Try to match an incoming message to a pipeline response. @@ -299,7 +268,7 @@ impl FilterHeadersPipeline { } // Match by stop_hash - the response includes it - if !self.coordinator.is_in_flight(&cfheaders.stop_hash) { + if !self.batch_starts.contains_key(&cfheaders.stop_hash) { return None; } @@ -312,7 +281,6 @@ impl FilterHeadersPipeline { /// Returns `Some(data)` if this response is the next expected and should /// be processed immediately. Returns `None` if buffered for later. pub(super) fn receive(&mut self, start_height: u32, data: CFHeaders) -> Option { - self.coordinator.receive(&data.stop_hash); self.batch_starts.remove(&data.stop_hash); if start_height == self.next_expected { diff --git a/dash-spv/src/sync/filter_headers/sync_manager.rs b/dash-spv/src/sync/filter_headers/sync_manager.rs index 9cf2b519e..f4fd03600 100644 --- a/dash-spv/src/sync/filter_headers/sync_manager.rs +++ b/dash-spv/src/sync/filter_headers/sync_manager.rs @@ -34,12 +34,6 @@ impl SyncManager for FilterHeade &[crate::network::MessageType::CfHeaders] } - fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { - if let crate::network::RequestKey::CfHeaders(stop_hash) = key { - self.pipeline.mark_on_flight(stop_hash); - } - } - fn on_disconnect(&mut self) { self.pipeline = FilterHeadersPipeline::default(); self.checkpoint_start_height = None; @@ -64,8 +58,14 @@ impl SyncManager for FilterHeade let mut events = Vec::new(); + // Correlated to an outstanding request (whether processed now or buffered + // out-of-order): stop the network manager tracking it for timeout. + let stop_hash = cfheaders.stop_hash; + let received = self.pipeline.receive(start_height, cfheaders); + network.request_answered(crate::network::RequestKey::CfHeaders(stop_hash)).await; + // Try to receive (may buffer if out of order) - if let Some(data) = self.pipeline.receive(start_height, cfheaders) { + if let Some(data) = received { // In order - process immediately let headers = self.process_cfheaders(&data, start_height).await?; let count = headers.len() as u32; @@ -183,8 +183,8 @@ impl SyncManager for FilterHeade } async fn tick(&mut self, network: &Arc) -> SyncResult> { - // Re-issue any timed-out requests, then send pending. - self.pipeline.handle_timeouts(std::time::Duration::from_secs(20)); + // Timeouts/retry are the network manager's job now (it re-queues via + // `requeue_timed_out`). // In-memory-mode safety net: if a BlockHeadersInMemory event was dropped // (lagged bus), the pipeline stalls with a gap at next_expected. Recover diff --git a/dash-spv/src/sync/filters/pipeline.rs b/dash-spv/src/sync/filters/pipeline.rs index 0c4680d33..3404d1fca 100644 --- a/dash-spv/src/sync/filters/pipeline.rs +++ b/dash-spv/src/sync/filters/pipeline.rs @@ -1,7 +1,7 @@ //! CFilters pipeline implementation. //! //! Handles pipelined download of compact block filters (BIP 157/158). -//! Tracks batch start heights with a DownloadCoordinator, plus +//! Declares wanted batches to the network manager's request broker, plus //! per-batch tracking for individual filter responses. //! //! Filters are buffered in a HashMap until the entire batch @@ -14,10 +14,9 @@ use dashcore::network::message::NetworkMessage; use dashcore::network::message_filter::GetCFilters; use dashcore::BlockHash; -use crate::error::{SyncError, SyncResult}; +use crate::error::SyncResult; use crate::network::PeerNetworkManager; use crate::storage::BlockHeaderStorage; -use crate::sync::download_coordinator::DownloadCoordinator; use crate::sync::filters::batch::FiltersBatch; use crate::sync::filters::batch_tracker::BatchTracker; @@ -26,17 +25,20 @@ const FILTER_BATCH_SIZE: u32 = 1000; /// Pipeline for downloading compact block filters. /// -/// Tracks batch start heights with a DownloadCoordinator, with -/// BatchTracker for tracking individual filters within each batch. -/// -/// Filters are buffered until the entire batch is complete, then returned -/// via `take_completed_batches()` for verification and matching. +/// Holds no request queue of its own: it declares the batches it wants to the +/// network manager (the broker de-duplicates, paces, times out and retries) and +/// buffers each batch's filters in a [`BatchTracker`] until it is complete. A +/// batch is "wanted" for exactly as long as its tracker sits in `batch_trackers` +/// (removed once the batch completes). #[derive(Debug)] pub(super) struct FiltersPipeline { - /// Coordinates pending/in-flight batch start heights. - coordinator: DownloadCoordinator, /// Tracks individual filter receipts per batch (start_height -> tracker). + /// Doubles as the "which batches are still wanted?" set. batch_trackers: HashMap, + /// Cached stop hash per wanted batch (start_height -> stop_hash), so + /// re-declaring the wanted set each tick doesn't re-read storage per batch. + /// Filled lazily in `send_pending` as headers become available. + batch_stops: HashMap, /// Completed filter batches. completed_batches: BTreeSet, /// Target height for sync. @@ -57,8 +59,8 @@ impl FiltersPipeline { /// Create a new CFilters pipeline. pub(super) fn new() -> Self { Self { - coordinator: DownloadCoordinator::new(), batch_trackers: HashMap::new(), + batch_stops: HashMap::new(), completed_batches: BTreeSet::new(), target_height: 0, filters_received: 0, @@ -66,9 +68,9 @@ impl FiltersPipeline { } } - /// Returns true if the pipeline has no in-flight or pending work. + /// Returns true if the pipeline has no wanted batches left. pub(super) fn is_idle(&self) -> bool { - self.coordinator.is_empty() + self.batch_trackers.is_empty() } /// Take completed batches with their buffered filter data for processing. @@ -78,19 +80,18 @@ impl FiltersPipeline { /// Initialize the pipeline for a sync range. /// - /// Pre-queues all batches for the range in the pending queue. + /// Creates a tracker for every batch in the range; `send_pending` then + /// declares them to the broker. pub(super) fn init(&mut self, start_height: u32, target_height: u32) { - self.coordinator.clear(); self.batch_trackers.clear(); + self.batch_stops.clear(); self.completed_batches.clear(); self.target_height = target_height; self.highest_received = start_height.saturating_sub(1); self.filters_received = 0; - // Pre-queue all batches let mut current = start_height; while current <= target_height { - self.coordinator.enqueue([current]); let batch_end = (current + FILTER_BATCH_SIZE - 1).min(target_height); self.batch_trackers.insert(current, BatchTracker::new(batch_end)); current = batch_end + 1; @@ -99,7 +100,7 @@ impl FiltersPipeline { /// Extend the target height without resetting pipeline state. /// - /// Queues additional batches from the old target boundary to the new target. + /// Adds trackers for the batches from the old target boundary to the new target. pub(super) fn extend_target(&mut self, new_target: u32) { if new_target <= self.target_height { return; @@ -108,81 +109,67 @@ impl FiltersPipeline { let old_target = self.target_height; self.target_height = new_target; - // Queue new batches from (old_target + 1) to new_target let mut current = old_target + 1; while current <= new_target { - self.coordinator.enqueue([current]); let batch_end = (current + FILTER_BATCH_SIZE - 1).min(new_target); self.batch_trackers.insert(current, BatchTracker::new(batch_end)); current = batch_end + 1; } } - /// Re-queue GetCFilters batches whose responses never fully arrived (peer - /// died or filters lost), so the next `send_pending` re-issues them. - pub(super) fn handle_timeouts(&mut self, timeout: std::time::Duration) { - let timed_out = self.coordinator.take_timed_out(timeout); - if !timed_out.is_empty() { - tracing::warn!("Filters: re-queuing {} timed-out batch(es)", timed_out.len()); - self.coordinator.enqueue(timed_out); - } - } - - /// Start the response timeout for a batch the router just put on the wire. - pub(super) fn mark_on_flight(&mut self, start_height: u32) { - self.coordinator.mark_on_flight(&[start_height]); - } - - /// Send pending filter requests up to the concurrency limit. + /// Declare every wanted batch to the network manager. + /// + /// Resolves (and caches) each batch's stop hash from storage the first time + /// it becomes available, then declares all resolved batches to the broker, + /// which de-duplicates in-flight ones. Batches whose stop header isn't stored + /// yet are simply skipped this round and retried next tick. pub(super) async fn send_pending( &mut self, network: &Arc, storage: &impl BlockHeaderStorage, ) -> SyncResult { - let count = self.coordinator.available_to_send(); - if count == 0 { + if self.batch_trackers.is_empty() { return Ok(0); } - let start_heights = self.coordinator.take_pending(count); - let mut sent = 0; - - for start_height in start_heights { - let batch_end = match self.batch_trackers.get(&start_height) { - Some(tracker) => tracker.end_height(), - None => { - return Err(SyncError::InvalidState(format!( - "missing batch tracker for start_height {}", - start_height - ))); + // Resolve stop hashes for any wanted batch we haven't cached yet. + let uncached: Vec<(u32, u32)> = self + .batch_trackers + .iter() + .filter(|(start, _)| !self.batch_stops.contains_key(start)) + .map(|(&start, tracker)| (start, tracker.end_height())) + .collect(); + for (start, batch_end) in uncached { + match storage.get_header(batch_end).await { + Ok(Some(h)) => { + self.batch_stops.insert(start, *h.hash()); } - }; - - // Get stop hash for this batch. If the header isn't available yet, - // re-queue for the next tick instead of losing the batch permanently. - let stop_hash = match storage.get_header(batch_end).await { - Ok(Some(h)) => *h.hash(), Ok(None) => { tracing::debug!( - "Header at height {} not yet available, re-queuing filter batch {}", + "Header at height {} not yet available, deferring filter batch {}", batch_end, - start_height + start ); - self.coordinator.enqueue([start_height]); - continue; } Err(e) => { - tracing::warn!( - "Error reading header at height {}, re-queuing filter batch {}: {}", - batch_end, - start_height, - e - ); - self.coordinator.enqueue([start_height]); - continue; + tracing::warn!("Error reading header at height {}: {}", batch_end, e); } - }; + } + } + // Declare every wanted batch whose stop hash is known, lowest-first for a + // deterministic fan-out. The broker de-duplicates, so re-declaring one + // already in flight is a no-op. + let mut ready: Vec<(u32, BlockHash)> = self + .batch_stops + .iter() + .filter(|(start, _)| self.batch_trackers.contains_key(start)) + .map(|(&start, &stop)| (start, stop)) + .collect(); + ready.sort_unstable_by_key(|(start, _)| *start); + + let n = ready.len(); + for (start_height, stop_hash) in ready { network .send(NetworkMessage::GetCFilters(GetCFilters { filter_type: 0u8, @@ -190,26 +177,17 @@ impl FiltersPipeline { stop_hash, })) .await; - - self.coordinator.mark_sent(&[start_height]); - - tracing::debug!( - "Sent GetCFilters: {} to {} ({} active batches)", - start_height, - batch_end, - self.coordinator.active_count() - ); - - sent += 1; } - Ok(sent) + Ok(n) } /// Handle a received CFilter message with filter data. /// /// Buffers the filter data for batch verification and wallet matching. - /// Returns `Some(height)` when a batch completes, `None` otherwise. + /// Returns `Some(batch_start)` when a batch completes (the `GetCFilters` + /// request key, so the caller can tell the network manager it was answered), + /// `None` otherwise. pub(super) fn receive_with_data( &mut self, height: u32, @@ -245,13 +223,11 @@ impl FiltersPipeline { let filters = self.batch_trackers.get_mut(&batch_start).map(|t| t.take_filters()).unwrap_or_default(); + // Out of the wanted set: drop its tracker and cached stop hash so the next + // `send_pending` won't re-declare it (the manager also tells the broker it + // was answered, stopping any in-flight retry). self.batch_trackers.remove(&batch_start); - // If the batch wasn't in-flight, it may still be sitting in the pending - // queue (e.g. completed via a late response); drop it so the next - // `send_pending` doesn't resurrect a finished batch. - if !self.coordinator.receive(&batch_start) { - self.coordinator.cancel_pending(&batch_start); - } + self.batch_stops.remove(&batch_start); tracing::info!( "Filter batch {}-{} complete ({} filters)", @@ -262,7 +238,7 @@ impl FiltersPipeline { let batch = FiltersBatch::new(batch_start, end_height, filters); self.completed_batches.insert(batch); - Some(height) + Some(batch_start) } /// Find which batch a filter height belongs to. diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 28e3737e3..2ca067c14 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -39,12 +39,6 @@ impl< &[crate::network::MessageType::CFilter] } - fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { - if let crate::network::RequestKey::CFilters(start_height) = key { - self.filter_pipeline.mark_on_flight(*start_height); - } - } - /// Keep `active_batches`, the block-match tracker, pending verified /// batches, and the filter pipeline's per-batch trackers across the /// disconnect. In-flight `getcfilters` slots are left untouched; reissuing @@ -140,15 +134,17 @@ impl< self.filter_pipeline.receive_with_data(h, cfilter.block_hash, &cfilter.filter); // A completed batch == one `getcfilters` request fully answered: free that - // peer's in-flight unit (the reader skips per-`cfilter` decrements). - if batch_completed.is_some() { + // peer's in-flight unit (the reader skips per-`cfilter` decrements) and stop + // the network manager tracking the batch for timeout. + if let Some(batch_start) = batch_completed { network.request_completed(peer, 1).await; + network.request_answered(crate::network::RequestKey::CFilters(batch_start)).await; } - // Send more requests if there are free slots - let header_storage = self.header_storage.read().await; - self.filter_pipeline.send_pending(network, &*header_storage).await?; - drop(header_storage); + // No `send_pending` here: the whole wanted set is already declared to the + // broker, which paces it out as capacity frees. Re-declaring per received + // `cfilter` would re-scan every wanted batch on the hot path (O(wanted) per + // message). The tick re-declares to pick up newly-available batches. let events = self.store_and_match_batches().await?; Ok(events) @@ -295,10 +291,8 @@ impl< return Ok(vec![]); } - // Re-issue any timed-out GetCFilters, then send pending (decoupled from - // processing). Filters yield ~1000 messages per batch, so use a longer - // timeout than the 1:1 request phases. - self.filter_pipeline.handle_timeouts(std::time::Duration::from_secs(30)); + // Timeouts/retry are the network manager's job now (it re-queues via + // `requeue_timed_out`); just push pending (decoupled from processing). let header_storage = self.header_storage.read().await; self.filter_pipeline.send_pending(network, &*header_storage).await?; drop(header_storage); diff --git a/dash-spv/src/sync/masternodes/pipeline.rs b/dash-spv/src/sync/masternodes/pipeline.rs index 0b4752700..e1c8eaad7 100644 --- a/dash-spv/src/sync/masternodes/pipeline.rs +++ b/dash-spv/src/sync/masternodes/pipeline.rs @@ -8,20 +8,20 @@ use std::sync::Arc; use crate::error::SyncResult; use crate::network::PeerNetworkManager; -use crate::sync::download_coordinator::DownloadCoordinator; use dashcore::network::message::NetworkMessage; use dashcore::network::message_sml::{GetMnListDiff, MnListDiff}; use dashcore::BlockHash; /// Pipeline for downloading MnListDiff messages for quorum validation. /// -/// Tracks requests by target block_hash using a download coordinator, with a -/// HashMap to store the base hash for each request. +/// Holds no request queue of its own: it declares the diffs it wants to the +/// network manager (the broker de-duplicates and retries) and correlates the +/// responses. A diff is "wanted" for exactly as long as its target sits in +/// `base_hashes` (removed once its response arrives). #[derive(Debug)] pub(super) struct MnListDiffPipeline { - /// Coordinates pending and in-flight requests, keyed by target block_hash. - coordinator: DownloadCoordinator, - /// Maps target_hash -> base_hash for each request. + /// Wanted requests: target_hash -> base_hash. Doubles as the "is this diff + /// wanted?" set for validating arrivals. base_hashes: HashMap, } @@ -35,14 +35,12 @@ impl MnListDiffPipeline { /// Create a new MnListDiff pipeline. pub(super) fn new() -> Self { Self { - coordinator: DownloadCoordinator::new(), base_hashes: HashMap::new(), } } /// Clear all state. pub(super) fn clear(&mut self) { - self.coordinator.clear(); self.base_hashes.clear(); } @@ -51,7 +49,6 @@ impl MnListDiffPipeline { /// Each request is a (base_hash, target_hash) pair. pub(super) fn queue_requests(&mut self, requests: Vec<(BlockHash, BlockHash)>) { for (base_hash, target_hash) in requests { - self.coordinator.enqueue([target_hash]); self.base_hashes.insert(target_hash, base_hash); } @@ -60,54 +57,35 @@ impl MnListDiffPipeline { } } - /// Send pending requests. - /// - /// Returns the number of requests sent. + /// Declare every wanted diff to the network manager. The broker + /// de-duplicates, so re-declaring one already in flight is a no-op. pub(super) async fn send_pending( &mut self, network: &Arc, ) -> SyncResult<()> { - let count = self.coordinator.available_to_send(); - if count == 0 { + if self.base_hashes.is_empty() { return Ok(()); } - - let target_hashes = self.coordinator.take_pending(count); - - for target_hash in target_hashes { - let Some(&base_hash) = self.base_hashes.get(&target_hash) else { - tracing::warn!("Missing base hash for target {}, skipping", target_hash); - continue; - }; - + // Deterministic order (target hash) so requests fan out the same way + // run-to-run rather than in random HashMap order. The broker + // de-duplicates, so re-declaring one already in flight is a no-op. + let mut requests: Vec<(BlockHash, BlockHash)> = + self.base_hashes.iter().map(|(target, base)| (*target, *base)).collect(); + requests.sort_unstable_by_key(|(target, _)| *target); + for (block_hash, base_block_hash) in requests { network .send(NetworkMessage::GetMnListD(GetMnListDiff { - base_block_hash: base_hash, - block_hash: target_hash, + base_block_hash, + block_hash, })) .await; - self.coordinator.mark_sent(&[target_hash]); - - tracing::debug!( - "Sent GetMnListDiff: base={}, target={} ({} active, {} pending)", - base_hash, - target_hash, - self.coordinator.active_count(), - self.coordinator.pending_count() - ); } - Ok(()) } - /// Check if response matches an in-flight request. + /// Check if response matches a wanted request. pub(super) fn match_response(&self, diff: &MnListDiff) -> bool { - self.coordinator.is_in_flight(&diff.block_hash) - } - - /// Start the response timeout for a request the router just put on the wire. - pub(super) fn mark_on_flight(&mut self, target_hash: &BlockHash) { - self.coordinator.mark_on_flight(&[*target_hash]); + self.base_hashes.contains_key(&diff.block_hash) } /// Receive a MnListDiff response. @@ -116,16 +94,14 @@ impl MnListDiffPipeline { pub(super) fn receive(&mut self, diff: &MnListDiff) -> bool { let target_hash = diff.block_hash; - if !self.coordinator.receive(&target_hash) { + if self.base_hashes.remove(&target_hash).is_none() { return false; } - self.base_hashes.remove(&target_hash); - tracing::debug!( "Received MnListDiff for {} ({} remaining)", target_hash, - self.coordinator.remaining() + self.base_hashes.len() ); true @@ -133,6 +109,6 @@ impl MnListDiffPipeline { /// Check if pipeline has no pending work. pub(super) fn is_complete(&self) -> bool { - self.coordinator.is_empty() + self.base_hashes.is_empty() } } diff --git a/dash-spv/src/sync/masternodes/sync_manager.rs b/dash-spv/src/sync/masternodes/sync_manager.rs index 2198b161d..cd06cc928 100644 --- a/dash-spv/src/sync/masternodes/sync_manager.rs +++ b/dash-spv/src/sync/masternodes/sync_manager.rs @@ -228,12 +228,6 @@ impl SyncManager for MasternodesManager { &[MessageType::MnListDiff, MessageType::QrInfo] } - fn mark_on_flight(&mut self, key: &crate::network::RequestKey) { - if let crate::network::RequestKey::MnListDiff(target_hash) = key { - self.sync_state.mnlistdiff_pipeline.mark_on_flight(target_hash); - } - } - fn on_disconnect(&mut self) { self.sync_state.clear_pending(); self.sync_state.qrinfo_retry_count = 0; @@ -343,6 +337,12 @@ impl SyncManager for MasternodesManager { return Ok(vec![]); } + // Correlated to an outstanding request: stop the broker tracking it + // for timeout (covers every path below, all of which consume it). + network + .request_answered(crate::network::RequestKey::MnListDiff(diff.block_hash)) + .await; + tracing::debug!("Processing MnListDiff message for {}", diff.block_hash); // Get target height from storage diff --git a/dash-spv/src/sync/mod.rs b/dash-spv/src/sync/mod.rs index 9d97ebe82..8645a8540 100644 --- a/dash-spv/src/sync/mod.rs +++ b/dash-spv/src/sync/mod.rs @@ -3,7 +3,6 @@ mod block_headers; mod blocks; mod chainlock; -mod download_coordinator; mod events; mod filter_headers; mod filters; diff --git a/dash-spv/src/sync/sync_manager.rs b/dash-spv/src/sync/sync_manager.rs index bea6d528c..1892d0802 100644 --- a/dash-spv/src/sync/sync_manager.rs +++ b/dash-spv/src/sync/sync_manager.rs @@ -1,5 +1,5 @@ use crate::error::SyncResult; -use crate::network::{NetworkEvent, PeerNetworkManager, RequestKey}; +use crate::network::{NetworkEvent, PeerNetworkManager}; use crate::sync::{ BlockHeadersProgress, BlocksProgress, ChainLockProgress, FilterHeadersProgress, FiltersProgress, InstantSendProgress, ManagerIdentifier, MasternodesProgress, MempoolProgress, @@ -84,7 +84,6 @@ impl std::fmt::Display for NetworkEvent { } => write!(f, "PeersUpdated({connected_count} peers)"), NetworkEvent::PeerConnected(addr) => write!(f, "PeerConnected({addr})"), NetworkEvent::PeerDisconnected(addr) => write!(f, "PeerDisconnected({addr})"), - NetworkEvent::RequestOnFlight(_) => write!(f, "RequestOnFlight"), } } } @@ -116,11 +115,6 @@ pub(super) async fn default_handle_network_event( return manager.start_sync(network).await; } } - // A request just went on the wire: let the owning pipeline start its - // response timeout from now (see `mark_on_flight`). - if let NetworkEvent::RequestOnFlight(key) = event { - manager.mark_on_flight(key); - } Ok(vec![]) } @@ -234,12 +228,6 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { default_handle_network_event(self, event, network).await } - /// Start the response timeout for a request the network manager just put on - /// the wire. Default no-op; each pipeline-owning manager overrides it to - /// forward the matching `RequestKey` variant to its coordinator's - /// `mark_on_flight`. Keeps retry counting from actual send, not from queue. - fn mark_on_flight(&mut self, _key: &RequestKey) {} - /// Flush any durable state before the task exits on shutdown. Default no-op; /// managers holding a write-back cache (the filters manager's scan frontier) /// override it so a graceful stop leaves that state on disk for the next run. @@ -375,10 +363,12 @@ pub trait SyncManager: Send + Sync + std::fmt::Debug { } } Err(broadcast::error::RecvError::Lagged(n)) => { - // Network-event bus overflowed (RequestOnFlight is - // high-volume). Dropping one is harmless — the request - // falls back to its queue-grace timeout — so keep - // running instead of killing the manager. + // Network-event bus overflowed. A dropped + // `RequestTimedOut` would strand that request in the + // pipeline's in-flight set, but the bus now carries only + // low-volume events (peer churn + actual timeouts), so + // lagging 4096 behind is not realistic; keep running + // rather than kill the manager over a warning. tracing::warn!("{} lagged network events, skipped {}", identifier, n); } Err(error) => { From 4b918a600dddb87d37b7b699ea98fbf2efb586ce Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Fri, 17 Jul 2026 07:06:57 +0000 Subject: [PATCH 16/25] feat(dash-spv): per-peer request dyn limit --- dash-spv/src/network/manager.rs | 151 +++++++++++++++++++++++++++++--- dash-spv/src/network/peer.rs | 34 ++++++- 2 files changed, 170 insertions(+), 15 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 706b900dc..b05738ea4 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -697,9 +697,37 @@ async fn route_tick( sent } -/// Sizes the global in-flight budget from MEASURED download capacity — no fixed -/// magic number, per the design goal of estimating how many requests the current -/// network can absorb before it saturates. +/// Per-peer state the bandwidth controller carries across windows to size each +/// connection's in-flight cap independently, by Little's Law over that peer's OWN +/// completion stream (rather than an even split of the global budget). +#[derive(Default, Clone, Copy)] +struct PeerCapState { + /// Cumulative completions/service-ns at the last window, to diff against. + last_count: u64, + last_total_ns: u64, + /// Cumulative bytes downloaded from this peer at the last window, for its + /// per-window throughput. + last_bytes: u64, + /// Uncongested service-time baseline in seconds (the peer's min `W`), 0 until + /// first measured. Little's Law targets `L = λ · min_W`. + min_w: f64, + /// Smoothed cap, so it doesn't jitter window to window. + cap_ema: f64, +} + +/// Sizes the host's GLOBAL in-flight budget from MEASURED download throughput and +/// each PEER's cap from its own completion stream — no fixed magic number, per the +/// design goal of estimating how many requests the current network can absorb +/// before it saturates. +/// +/// Two levels, both measured: +/// - The GLOBAL budget (host downlink) hill-climbs on bytes/s read off the sockets +/// (see below). It is the ceiling the host can reach with all peers combined. +/// - Each PEER's cap is `L = λ · min_W` (Little's Law) from THAT peer's completion +/// rate `λ` and uncongested service time `min_W`, so a fast peer earns a high cap +/// and a slow one a low cap. `route_tick` binds `min(Σ per-peer room, global +/// room)`, so whichever is the real bottleneck — the peers or the host link — +/// limits each round. /// /// The estimate is Little's Law applied to downloads: the number of requests in /// flight that sustains a completion rate `λ` at an uncongested per-request @@ -731,6 +759,10 @@ fn spawn_bandwidth_controller( const REPROBE: u32 = 8; // plateau windows to hold before nudging the cap up const IDLE_BPS: f64 = 1.0e6; // downlink under 1 MB/s = idle, hold the cap const EMA_ALPHA: f64 = 0.5; // smoothing for the noisy per-window rate + // Per-peer cap sizing (Little's Law over each peer's own completions). + const CAP_PROBE: f64 = 1.25; // aim a little past L to keep discovering headroom + const CAP_ALPHA: f64 = 0.3; // EMA smoothing for the per-peer cap + const W_INFLATE: f64 = 2.0; // W above min_W·this => congested, stop probing up let window_s = WINDOW.as_secs_f64(); tokio::spawn(async move { @@ -738,6 +770,8 @@ fn spawn_bandwidth_controller( let mut rate_ema = 0.0f64; // smoothed downlink bytes/s let mut last_rate = 0.0f64; // smoothed rate at the previous cap adjustment let mut hold = 0u32; // consecutive plateau windows + // Per-peer cap state across windows, keyed by peer address. + let mut peer_caps: HashMap = HashMap::new(); let mut ticker = tokio::time::interval(WINDOW); loop { tokio::select! { @@ -802,27 +836,117 @@ fn spawn_bandwidth_controller( } cap.store(new, Ordering::Relaxed); - // Spread the global budget evenly across peers with a slot of headroom, - // so the GLOBAL cap is the binding constraint (route_tick fills the - // emptiest peer first, so fast peers naturally serve more) while no - // single connection is over-committed beyond PEER_CEIL. - let per_peer = (new.div_ceil(npeers) + 1).clamp(FLOOR_PER_PEER, PEER_CEIL); + // Size each peer's cap INDEPENDENTLY from its own completion stream by + // Little's Law (`L = λ · min_W`), rather than an even split of the + // global budget. The global cap above stays the host ceiling — a peer + // whose measured cap sums past it is still bound by `global_room` in + // `route_tick`. A fast peer earns a high cap, a slow one a low cap; both + // clamp to [FLOOR_PER_PEER, PEER_CEIL]. + let (mut cap_min, mut cap_max, mut cap_sum) = (usize::MAX, 0usize, 0usize); + let mut inflight_sum = 0usize; // total requests on the wire right now { let g = connected.lock().await; + let live: HashSet = g.iter().map(|(p, _)| p.addr()).collect(); for (p, _) in g.iter() { - p.set_cap(per_peer); + let addr = p.addr(); + let (count, total_ns) = p.latency_totals(); + let now_bytes = p.bytes_read(); + let st = peer_caps.entry(addr).or_default(); + let dc = count.saturating_sub(st.last_count); + let dt = total_ns.saturating_sub(st.last_total_ns); + let d_bytes = now_bytes.saturating_sub(st.last_bytes); + st.last_count = count; + st.last_total_ns = total_ns; + st.last_bytes = now_bytes; + let rate = d_bytes as f64 / window_s; // this peer's downlink (bytes/s) + + let (lambda, w) = if dc == 0 { + // No completions this window: either idle (no work queued to + // it) or stalled — the timeout monitor kicks a stalled peer at + // REQUEST_TIMEOUT. Keep at least the floor so the router can + // hand it work to bootstrap/keep measuring, but don't grow blind. + st.cap_ema = st.cap_ema.max(FLOOR_PER_PEER as f64); + (0.0, 0.0) + } else { + let lambda = dc as f64 / window_s; // completions/sec + let w = (dt as f64 / dc as f64) / 1e9; // avg service time (s) + st.min_w = if st.min_w == 0.0 { + w + } else { + st.min_w.min(w) + }; + // Sustaining level, then probe a little past it — unless W has + // inflated past the uncongested baseline, which means this peer + // is already over-committed, so hold at the sustaining level. + let base = lambda * st.min_w; + let target = if w > st.min_w * W_INFLATE { + base + } else { + base * CAP_PROBE + }; + st.cap_ema = if st.cap_ema == 0.0 { + target + } else { + CAP_ALPHA * target + (1.0 - CAP_ALPHA) * st.cap_ema + }; + (lambda, w) + }; + + let cap_peer = (st.cap_ema.round() as usize).clamp(FLOOR_PER_PEER, PEER_CEIL); + p.set_cap(cap_peer); + + tracing::debug!( + target: "peer_speed", + "peer {}: cap={} in_flight={} lag={}ms lambda={:.1}/s W={:.0}ms rate={:.2} MB/s", + addr, + cap_peer, + p.in_flight(), + p.lag_ms(), + lambda, + w * 1e3, + rate / 1e6, + ); + + cap_min = cap_min.min(cap_peer); + cap_max = cap_max.max(cap_peer); + cap_sum += cap_peer; + inflight_sum += p.in_flight(); } + // Drop state for peers that have disconnected. + peer_caps.retain(|addr, _| live.contains(addr)); + } + if cap_min == usize::MAX { + cap_min = 0; } + // `bind` names the constraint the router is hitting: `global` if the + // host budget is the smaller room, `peers` if the summed per-peer caps + // are, `work` if neither is full (queue-limited or peers just slow). If + // the downlink is saturated the global cap should hold at the knee and + // `bind=global`. + let bind = if inflight_sum >= new.min(cap_sum) { + if new <= cap_sum { + "global" + } else { + "peers" + } + } else { + "work" + }; tracing::debug!( - target: "dash_spv::network", - "bandwidth: {:.1} MB/s (ema {:.1}) | {} | global cap={} | peers={} per-peer cap={}", + target: "peer_speed", + "bandwidth: {:.1} MB/s (ema {:.1}) | {} bind={} | global_cap={} sum_peer_cap={} in_flight={} | peers={} per-peer cap min/avg/max={}/{}/{}", dl_rate / 1e6, rate_ema / 1e6, action, + bind, new, + cap_sum, + inflight_sum, npeers, - per_peer, + cap_min, + cap_sum / npeers.max(1), + cap_max, ); } }) @@ -985,8 +1109,9 @@ fn spawn_timeout_monitor( tracing::warn!( target: "dash_spv::network", - "request timeout: kicked {} peer(s), retried {} request(s)", + "request timeout: kicked {} peer(s) {:?}, retried {} request(s)", dropped, + culprits, reinject.len(), ); diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 5b92b66d0..d6714fae1 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -44,6 +44,13 @@ impl Latency { }; (count, avg_ms, max as f64 / 1e6) } + + /// Cumulative (completed request count, total service-time nanoseconds). + /// The bandwidth controller diffs these across a window to get THIS peer's + /// completion rate and service time, sizing its in-flight cap by Little's Law. + fn totals(&self) -> (u64, u64) { + (self.count.load(Ordering::Relaxed), self.total_ns.load(Ordering::Relaxed)) + } } use dashcore::{ @@ -79,7 +86,11 @@ const USER_AGENT: &str = concat!("/dash-spv:", env!("CARGO_PKG_VERSION"), "/"); /// in-flight budget to ~90% of it). struct CountingReader { inner: R, + /// Host-wide download counter (all peers), the global bandwidth signal. bytes: Arc, + /// This connection's own download counter, so the controller can measure + /// per-peer throughput. + peer_bytes: Arc, } impl AsyncRead for CountingReader { @@ -91,8 +102,9 @@ impl AsyncRead for CountingReader { let before = buf.filled().len(); let r = std::pin::Pin::new(&mut self.inner).poll_read(cx, buf); if let std::task::Poll::Ready(Ok(())) = &r { - let n = buf.filled().len() - before; - self.bytes.fetch_add(n as u64, Ordering::Relaxed); + let n = (buf.filled().len() - before) as u64; + self.bytes.fetch_add(n, Ordering::Relaxed); + self.peer_bytes.fetch_add(n, Ordering::Relaxed); } r } @@ -119,6 +131,8 @@ pub struct ConnectedPeer { /// completion rate and service time, mirroring how the global budget is sized /// from our download rate. Fast peers earn a high cap, slow peers a low one. cap: Arc, + /// Cumulative bytes downloaded from THIS peer, for per-peer throughput. + bytes: Arc, /// Per-connection cancel token (child of the global shutdown). Cancelling it /// stops this peer's reader and closes the socket. Used to drop peers we /// probed but don't keep, so we only hold connections we actually use. @@ -191,6 +205,13 @@ impl ConnectedPeer { self.latency.snapshot() } + /// Cumulative (completed request count, total service-time ns) for this peer. + /// The bandwidth controller diffs these across a window to size this peer's + /// in-flight cap independently by Little's Law. + pub(crate) fn latency_totals(&self) -> (u64, u64) { + self.latency.totals() + } + /// Handshake round-trip latency in ms (0 if unmeasured). pub(crate) fn lag_ms(&self) -> u32 { self.lag_ms.load(Ordering::Relaxed) @@ -207,6 +228,12 @@ impl ConnectedPeer { self.cap.load(Ordering::Relaxed) } + /// Cumulative bytes downloaded from this peer. The controller diffs it across + /// a window for this peer's throughput. + pub(crate) fn bytes_read(&self) -> u64 { + self.bytes.load(Ordering::Relaxed) + } + /// Update this peer's measured in-flight capacity (called by the controller). pub(crate) fn set_cap(&self, n: usize) { self.cap.store(n, Ordering::Relaxed); @@ -271,11 +298,13 @@ impl DisconnectedPeer { NetworkError::ConnectionFailed(format!("Failed to connect to {}: {}", self.addr, e)) })?; + let peer_bytes = Arc::new(AtomicU64::new(0)); let (read_half, mut writer) = stream.into_split(); let mut reader = FramedRead::new( CountingReader { inner: read_half, bytes, + peer_bytes: peer_bytes.clone(), }, RawNetworkMessageCodec, ); @@ -379,6 +408,7 @@ impl DisconnectedPeer { latency, writer, cap, + bytes: peer_bytes, token, }) } From 4c83af355c047b01ab35b2f18f7af897397a8c0b Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Fri, 17 Jul 2026 08:24:24 +0000 Subject: [PATCH 17/25] fix: run.sh works on macOS in theory --- dash-spv-bench/run.sh | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/dash-spv-bench/run.sh b/dash-spv-bench/run.sh index 8c5cbde42..a1bc98058 100755 --- a/dash-spv-bench/run.sh +++ b/dash-spv-bench/run.sh @@ -172,25 +172,32 @@ expand_cpus() { CPU_PREFIX=() if [ -n "${BENCH_CPUS}" ]; then - command -v taskset >/dev/null 2>&1 || { - echo "Error: cpus is set ('${BENCH_CPUS}') but 'taskset' was not found (Linux/util-linux)." >&2 - exit 1 - } - ncpu="$(nproc)" + # Portable core count: getconf works on Linux and macOS; fall back to the + # BSD/Linux specific tools, then to 0 (= "could not determine"). + ncpu="$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 0)" bench_cores="$(expand_cpus "${BENCH_CPUS}" | sort -nu)" - for core in ${bench_cores}; do - if [ "${core}" -lt 0 ] || [ "${core}" -ge "${ncpu}" ]; then - echo "Error: cpu ${core} is out of range for this ${ncpu}-core host (valid: 0-$((ncpu - 1)))." >&2 - exit 1 - fi - done - peer_cores="" - for i in $(seq 0 $((ncpu - 1))); do - grep -qxF "${i}" <<<"${bench_cores}" || peer_cores="${peer_cores},${i}" - done - BENCH_PEER_CPUS="${peer_cores#,}" - CPU_PREFIX=(taskset -c "${BENCH_CPUS}") - echo "==> pinning the measured run to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" + if [ "${ncpu}" -gt 0 ]; then + for core in ${bench_cores}; do + if [ "${core}" -lt 0 ] || [ "${core}" -ge "${ncpu}" ]; then + echo "Error: cpu ${core} is out of range for this ${ncpu}-core host (valid: 0-$((ncpu - 1)))." >&2 + exit 1 + fi + done + peer_cores="" + for i in $(seq 0 $((ncpu - 1))); do + grep -qxF "${i}" <<<"${bench_cores}" || peer_cores="${peer_cores},${i}" + done + BENCH_PEER_CPUS="${peer_cores#,}" + fi + + # taskset pins the measured binary on Linux. macOS has no per-process CPU + # affinity CLI, so degrade gracefully + if command -v taskset >/dev/null 2>&1; then + CPU_PREFIX=(taskset -c "${BENCH_CPUS}") + echo "==> pinning the measured run to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" + else + echo "==> note: 'taskset' not found (e.g. macOS); running the measured binary UNPINNED${BENCH_PEER_CPUS:+ (docker peers still pinned to ${BENCH_PEER_CPUS})}" >&2 + fi fi # --- local: generate the peer compose (now that the peer cpuset is known) ------------------- From a24bc5337ffb7a5665920183b05f02e11e92598d Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Fri, 17 Jul 2026 08:52:34 +0000 Subject: [PATCH 18/25] fix(dash-spv): reject peers with missing services --- dash-spv/src/network/manager.rs | 41 +++++++++++++++++++++++++++++---- dash-spv/src/network/peer.rs | 19 +++++++++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index b05738ea4..226eb28a8 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -8,6 +8,7 @@ use std::{ time::{Duration, Instant}, }; +use dashcore::network::constants::ServiceFlags; use dashcore::network::message::NetworkMessage; use dashcore::network::message_blockdata::Inventory; use futures::future::join_all; @@ -205,6 +206,9 @@ pub struct PeerNetworkManager { /// reconnector so its `PeersUpdated` carries the real height. best_tip: Arc, max_peers: usize, + /// Service flags a peer must advertise to be kept (e.g. COMPACT_FILTERS when + /// filters are enabled). Checked at handshake; `NONE` keeps every peer. + required_services: ServiceFlags, /// Total bytes read from all peers. Held so `start` can hand it to the peers it connects. bytes: Arc, // Cancelled by `stop()` to tear down the router, pump and every peer reader. @@ -264,6 +268,13 @@ impl PeerNetworkManager { pub async fn new(config: &ClientConfig) -> Self { let discoverer = Arc::new(Mutex::new(PeerDiscoverer::new(config))); let max_peers = config.max_peers.max(1) as usize; + // A filter-syncing client can only use peers that serve compact filters + // (BIP157 — the same flag also covers compact filter headers). + let required_services = if config.enable_filters { + ServiceFlags::COMPACT_FILTERS + } else { + ServiceFlags::NONE + }; let connected_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); let other_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); @@ -306,6 +317,7 @@ impl PeerNetworkManager { events_tx.clone(), best_tip.clone(), requests.clone(), + required_services, ); spawn_timeout_monitor( @@ -332,6 +344,7 @@ impl PeerNetworkManager { events_tx.clone(), best_tip.clone(), max_peers, + required_services, ); PeerNetworkManager { @@ -345,6 +358,7 @@ impl PeerNetworkManager { events_tx, best_tip, max_peers, + required_services, bytes, shutdown, } @@ -367,6 +381,7 @@ impl PeerNetworkManager { &self.inbound_tx, &self.shutdown, &self.bytes, + self.required_services, ) .await; @@ -523,6 +538,7 @@ fn spawn_router( events: broadcast::Sender, best_tip: Arc, requests: Registry, + required_services: ServiceFlags, ) -> JoinHandle<()> { tokio::spawn(async move { loop { @@ -558,6 +574,7 @@ fn spawn_router( &best_tip, ADD_PEERS_BATCH, MAX_CONNECTED_PEERS, + required_services, ) .await; } @@ -970,6 +987,7 @@ fn spawn_reconnector( events: broadcast::Sender, best_tip: Arc, max_peers: usize, + required_services: ServiceFlags, ) -> JoinHandle<()> { tokio::spawn(async move { let mut ticker = tokio::time::interval(RECONNECT_CHECK); @@ -1005,8 +1023,16 @@ fn spawn_reconnector( let before = connected.lock().await.len(); add_peers( - &connected, &others, &inbound, &shutdown, &bytes, &events, &best_tip, deficit, + &connected, + &others, + &inbound, + &shutdown, + &bytes, + &events, + &best_tip, + deficit, max_peers, + required_services, ) .await; let after = connected.lock().await.len(); @@ -1135,6 +1161,7 @@ async fn add_peers( best_tip: &Arc, batch: usize, max_peers: usize, + required_services: ServiceFlags, ) { // Never grow past the operating cap; only backfill the deficit toward it. let deficit = max_peers.saturating_sub(connected.lock().await.len()); @@ -1149,7 +1176,9 @@ async fn add_peers( let mut added = 0; for candidate in candidates { - if let Ok(peer) = candidate.connect(inbound.clone(), shutdown.clone(), bytes.clone()).await + if let Ok(peer) = candidate + .connect(inbound.clone(), shutdown.clone(), bytes.clone(), required_services) + .await { let addr = peer.addr(); // Keep `best_tip` current the moment a peer arrives: a newcomer may @@ -1200,6 +1229,7 @@ fn retire_drained(peer: ConnectedPeer, shutdown: CancellationToken) { }); } +#[allow(clippy::too_many_arguments)] async fn probe_and_select( discoverer: &mut PeerDiscoverer, connected: &Mutex>, @@ -1208,6 +1238,7 @@ async fn probe_and_select( inbound: &UnboundedSender, shutdown: &CancellationToken, bytes: &Arc, + required_services: ServiceFlags, ) -> u32 { const CONNECT_CHUNK: usize = 16; // bounded concurrent handshakes per round const FAST_LAG_MS: u32 = 100; // prefer peers that ping under this @@ -1239,9 +1270,9 @@ async fn probe_and_select( let take = CONNECT_CHUNK.min(candidates.len()); attempted += take; let batch: Vec = candidates.drain(..take).collect(); - let results = join_all( - batch.into_iter().map(|c| c.connect(inbound.clone(), shutdown.clone(), bytes.clone())), - ) + let results = join_all(batch.into_iter().map(|c| { + c.connect(inbound.clone(), shutdown.clone(), bytes.clone(), required_services) + })) .await; for peer in results.into_iter().filter_map(Result::ok) { tip = tip.max(peer.version().start_height.max(0) as u32); diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index d6714fae1..2967a9b16 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -293,6 +293,7 @@ impl DisconnectedPeer { inbound: UnboundedSender, shutdown: CancellationToken, bytes: Arc, + required_services: ServiceFlags, ) -> NetworkResult { let stream = TcpStream::connect(&self.addr).await.map_err(|e| { NetworkError::ConnectionFailed(format!("Failed to connect to {}: {}", self.addr, e)) @@ -344,6 +345,24 @@ impl DisconnectedPeer { let version = peer_version.ok_or(NetworkError::PeerDisconnected)?; + // Only keep peers that advertise the services we need. For a filter-syncing + // client that means COMPACT_FILTERS (BIP157) — the SAME flag gates cfilters + // AND cfheaders. A peer without it serves headers/blocks fine but silently + // ignores `getcfilters`, so its slots would stall until the request times + // out; drop it now rather than waste it. `has(NONE)` is always true, so an + // empty requirement (filters disabled) keeps every peer. + if !version.services.has(required_services) { + tracing::debug!( + target: "dash_spv::network", + "dropping {}: lacks required services {:?} (advertises {:?})", + self.addr, required_services, version.services + ); + return Err(NetworkError::ConnectionFailed(format!( + "peer {} lacks required services {:?}", + self.addr, required_services + ))); + } + // Announce sendheaders only after the handshake is fully complete. handshake_send(&mut writer, magic, NetworkMessage::SendHeaders).await?; From e2c69520e3e2846e71ffb63017a369efb50f71c6 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Fri, 17 Jul 2026 11:52:45 +0000 Subject: [PATCH 19/25] timeout per iveness --- dash-spv/src/network/manager.rs | 171 ++++++++++++------ dash-spv/src/sync/filters/manager.rs | 2 +- dash-spv/src/sync/filters/sync_manager.rs | 14 +- .../tests/dashd_sync/tests_multi_wallet.rs | 20 +- 4 files changed, 143 insertions(+), 64 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 226eb28a8..5896511c1 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -41,13 +41,15 @@ const RECONNECT_CHECK: Duration = Duration::from_secs(2); const STALL_CHECK: Duration = Duration::from_secs(5); /// A request unanswered for this long is treated as dead: the timeout monitor -/// re-queues it to another peer and kicks the peer that ignored it. +/// re-queues it to another peer and kicks the peer sitting on it. /// -/// Uniform across request types. Blocks are paced through the same budget and a -/// 2MB block over a weak link legitimately takes a while, but a peer that has -/// answered nothing in 30s is not worth waiting on — the reconnector refills the -/// slot with a fresh peer faster than a dead one would ever recover. -const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// Deliberately aggressive. A cfilter batch normally completes in well under a +/// second, so a peer holding a request for 10s is slow enough to be worth +/// dropping — the reconnector refills the slot with a fresh peer faster than a +/// laggard recovers, and we would rather churn a slow peer than let it drag its +/// in-flight slots. (The per-peer AIMED already throttles a merely-slow-but- +/// responding peer to the floor; this catches the ones that stop answering.) +const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); /// How often the timeout monitor scans the outstanding-request registry. const TIMEOUT_CHECK: Duration = Duration::from_secs(1); @@ -185,8 +187,6 @@ enum ReqState { struct OnWire { /// The peer the request was routed to. peer: SocketAddr, - /// When the request actually left the router for that peer. - sent_at: Instant, /// The exact message, re-queued verbatim on timeout (requests are /// self-contained, so re-sending the same bytes is a valid retry). msg: NetworkMessage, @@ -303,6 +303,7 @@ impl PeerNetworkManager { connected_peers.clone(), events_tx.clone(), msg_queue.clone(), + requests.clone(), shutdown.clone(), ); @@ -660,7 +661,7 @@ async fn route_tick( let mut unsent: Vec = Vec::new(); // Messages that made it onto the wire this round, recorded in the broker in one // lock acquisition after the send loop. - let mut on_wire: Vec<(NetworkMessage, SocketAddr, Instant)> = Vec::new(); + let mut on_wire: Vec<(NetworkMessage, SocketAddr)> = Vec::new(); let mut msgs = msgs.into_iter(); for msg in msgs.by_ref() { // Send to the peer with the most free measured capacity. @@ -674,8 +675,8 @@ async fn route_tick( }; if peer.send(&msg).await.is_ok() { sent += 1; - // Record which peer got it and when, so the monitor can time it out. - on_wire.push((msg, peer.addr(), Instant::now())); + // Record which peer got it, so the monitor can attribute a stall to it. + on_wire.push((msg, peer.addr())); } else { tracing::warn!(target: "dash_spv::network", "router: send to {} failed", peer.addr()); unsent.push(msg); @@ -686,7 +687,7 @@ async fn route_tick( if !on_wire.is_empty() { let mut reqs = requests.lock().await; - for (msg, peer, sent_at) in on_wire { + for (msg, peer) in on_wire { for key in request_keys(&msg) { // Transition Queued -> OnWire, keeping the message for retry. Skip // keys no longer present (cancelled while queued): the request went @@ -694,7 +695,6 @@ async fn route_tick( if let Some(slot) = reqs.get_mut(&key) { *slot = ReqState::OnWire(Box::new(OnWire { peer, - sent_at, msg: msg.clone(), })); } @@ -776,10 +776,13 @@ fn spawn_bandwidth_controller( const REPROBE: u32 = 8; // plateau windows to hold before nudging the cap up const IDLE_BPS: f64 = 1.0e6; // downlink under 1 MB/s = idle, hold the cap const EMA_ALPHA: f64 = 0.5; // smoothing for the noisy per-window rate - // Per-peer cap sizing (Little's Law over each peer's own completions). - const CAP_PROBE: f64 = 1.25; // aim a little past L to keep discovering headroom - const CAP_ALPHA: f64 = 0.3; // EMA smoothing for the per-peer cap - const W_INFLATE: f64 = 2.0; // W above min_W·this => congested, stop probing up + // Per-peer cap: AIMED driven purely by THIS peer's service-time (lag). There is + // NO hard per-peer request limit — a peer with headroom keeps growing, so we + // fill the peers we have instead of recruiting more. It only backs off when its + // own lag inflates past the uncongested baseline. + const CAP_GROW: f64 = 1.0; // additive increase per window while lag is flat + const CAP_BACKOFF: f64 = 0.8; // multiplicative decrease when lag inflates + const W_INFLATE: f64 = 1.5; // W above min_W·this => this peer is backing up let window_s = WINDOW.as_secs_f64(); tokio::spawn(async move { @@ -853,12 +856,12 @@ fn spawn_bandwidth_controller( } cap.store(new, Ordering::Relaxed); - // Size each peer's cap INDEPENDENTLY from its own completion stream by - // Little's Law (`L = λ · min_W`), rather than an even split of the - // global budget. The global cap above stays the host ceiling — a peer - // whose measured cap sums past it is still bound by `global_room` in - // `route_tick`. A fast peer earns a high cap, a slow one a low cap; both - // clamp to [FLOOR_PER_PEER, PEER_CEIL]. + // Size each peer's cap by AIMED on its OWN service time (lag): grow while + // its lag stays at the uncongested baseline (it has headroom), back off + // the moment its lag inflates (it is backing up). No hard per-peer + // request limit — a fast peer keeps growing so we fill the peers we have + // instead of forcing new connections while there is still room for work. + // The global cap above stays the host ceiling (route_tick binds by it). let (mut cap_min, mut cap_max, mut cap_sum) = (usize::MAX, 0usize, 0usize); let mut inflight_sum = 0usize; // total requests on the wire right now { @@ -892,20 +895,16 @@ fn spawn_bandwidth_controller( } else { st.min_w.min(w) }; - // Sustaining level, then probe a little past it — unless W has - // inflated past the uncongested baseline, which means this peer - // is already over-committed, so hold at the sustaining level. - let base = lambda * st.min_w; - let target = if w > st.min_w * W_INFLATE { - base - } else { - base * CAP_PROBE - }; - st.cap_ema = if st.cap_ema == 0.0 { - target + // AIMED on this peer's own lag: additive-increase while its + // service time sits at the uncongested baseline (headroom), + // multiplicative-decrease the moment it inflates (backing up). + if st.cap_ema == 0.0 { + st.cap_ema = FLOOR_PER_PEER as f64; + } else if w > st.min_w * W_INFLATE { + st.cap_ema = (st.cap_ema * CAP_BACKOFF).max(FLOOR_PER_PEER as f64); } else { - CAP_ALPHA * target + (1.0 - CAP_ALPHA) * st.cap_ema - }; + st.cap_ema += CAP_GROW; + } (lambda, w) }; @@ -1055,16 +1054,29 @@ fn spawn_reconnector( }) } -/// Time requests out and evict the peers that ignored them. +/// Time requests out and evict the peers that stalled on them. +/// +/// A peer is a culprit when it has in-flight work (`in_flight > 0`) AND no bytes +/// have arrived from it for a full [`REQUEST_TIMEOUT`] — i.e. it has gone silent +/// while owing us responses. Kicking is on peer liveness, not on any single +/// request's age: a healthy peer draining a deep backlog (the per-peer cap grows +/// with headroom, so we may have many requests queued at it) keeps sending bytes, +/// so its liveness clock keeps resetting even while its OLDEST request sits waiting +/// its turn. Only a peer that has stopped sending anything is worth dropping. +/// +/// Judging on the peer's own counters (not the broker registry) is deliberate: the +/// registry can desync from a peer's real `in_flight` — correlating a response +/// frees the registry key, while streaming in-flight units are freed on the peer +/// separately — so a wedged peer can pin `in_flight == cap` (starving the router, +/// which only sends to peers under cap) while showing zero registry entries. A +/// registry-based check would miss exactly that peer. /// -/// Scans the broker for any peer sitting on an on-wire request older than -/// [`REQUEST_TIMEOUT`] and kicks it immediately: every request routed to it is now +/// When one is found we kick it immediately: every request routed to it is now /// dead, so we pull ALL its on-wire entries, re-inject their messages (retry to a /// fresh peer), and drop the connection. Its in-flight slots die with it — no -/// separate reclaim — and the reconnector refills the peer set. -/// -/// Immediate kick is deliberate: a peer that has answered nothing for 30s is not -/// worth a second chance, and leaving it connected would just route the freed +/// separate reclaim — and the reconnector refills the peer set. Requests the +/// registry lost track of are re-driven by their pipeline's wanted set. Immediate +/// kick is deliberate: leaving a dead peer connected would just route the freed /// work straight back to it (the router fills the emptiest peer first). fn spawn_timeout_monitor( requests: Registry, @@ -1074,6 +1086,8 @@ fn spawn_timeout_monitor( ) -> JoinHandle<()> { tokio::spawn(async move { let mut ticker = tokio::time::interval(TIMEOUT_CHECK); + // Per-peer liveness: (bytes read at last progress, when it last rose). + let mut progress: HashMap = HashMap::new(); loop { tokio::select! { _ = shutdown.cancelled() => break, @@ -1081,17 +1095,34 @@ fn spawn_timeout_monitor( } let now = Instant::now(); - // Peers with at least one on-wire request past the timeout. + + // Liveness is judged on the peer itself, not on the broker registry. + // The registry can desync from a peer's real in-flight count (a request + // whose response we correlate frees the registry key, while streaming + // in-flight units are freed on the peer separately), so a wedged peer can + // hold `in_flight == cap` — blocking the router from sending it anything — + // while showing zero registry entries. Watching `in_flight` (has pending + // work) and `bytes_read` (is data still arriving — covers both completed + // requests and mid-stream progress) catches that: a peer with work + // outstanding whose byte counter is frozen for a full REQUEST_TIMEOUT is + // stuck and must be dropped, whatever the registry thinks. let culprits: HashSet = { - let reqs = requests.lock().await; - reqs.values() - .filter_map(|s| match s { - ReqState::OnWire(o) if now.duration_since(o.sent_at) > REQUEST_TIMEOUT => { - Some(o.peer) - } - _ => None, - }) - .collect() + let peers = connected.lock().await; + let live: HashSet = peers.iter().map(|(p, _)| p.addr()).collect(); + progress.retain(|addr, _| live.contains(addr)); + let mut culprits = HashSet::new(); + for (peer, _) in peers.iter() { + let addr = peer.addr(); + let bytes = peer.bytes_read(); + let entry = progress.entry(addr).or_insert((bytes, now)); + if bytes > entry.0 { + *entry = (bytes, now); + } + if peer.in_flight() > 0 && now.duration_since(entry.1) > REQUEST_TIMEOUT { + culprits.insert(addr); + } + } + culprits }; if culprits.is_empty() { continue; @@ -1328,6 +1359,7 @@ fn spawn_pump( connected: Arc>>, events: broadcast::Sender, queue: Arc, + requests: Registry, shutdown: CancellationToken, ) -> JoinHandle<()> { tokio::spawn(async move { @@ -1415,12 +1447,43 @@ fn spawn_pump( guard.retain(|(peer, _)| peer.addr() != addr); guard.len() }; + + // A peer vanishing takes its in-flight requests with it: every + // request routed to it is now dead and no one else is tracking it. + // Pull them back to Queued and re-inject the messages so the router + // retries them on a live peer — the timeout monitor only watches + // connected peers, so a request whose peer is already gone would + // otherwise leak in the registry forever. + let mut reinject: Vec = Vec::new(); + { + let mut reqs = requests.lock().await; + let keys: Vec = reqs + .iter() + .filter_map(|(k, s)| match s { + ReqState::OnWire(o) if o.peer == addr => Some(k.clone()), + _ => None, + }) + .collect(); + for key in keys { + if let Some(ReqState::OnWire(o)) = reqs.insert(key, ReqState::Queued) { + reinject.push(o.msg); + } + } + } + tracing::info!( target: "dash_spv::network", - "peer disconnected: {} | {} peers remaining", + "peer disconnected: {} | {} peers remaining | {} request(s) re-queued", addr, remaining, + reinject.len(), ); + + for msg in reinject { + queue.push(msg).await; + } + queue.notify.notify_one(); + let _ = events.send(NetworkEvent::PeerDisconnected(addr)); } } diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index fa7f267ea..05d647c0e 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -2026,7 +2026,7 @@ impl u32 { + pub(super) async fn download_ceiling(&self) -> u32 { let block_tip = self.header_storage.read().await.get_tip().await.map(|t| t.height()).unwrap_or(0); block_tip.max(self.progress.filter_header_tip_height()) diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 2ca067c14..7e796c741 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -291,8 +291,18 @@ impl< return Ok(vec![]); } - // Timeouts/retry are the network manager's job now (it re-queues via - // `requeue_timed_out`); just push pending (decoupled from processing). + // Keep the download ceiling at the BLOCK-header tip, extended here every + // tick rather than only on `FilterHeadersStored` events. A `getcfilters` + // only needs block hashes, so the download must not wait on the in-order + // cfheaders chain: coupling it to filter-header events starved the pipeline + // when block headers finished a step ahead of the last cfheaders batch, + // dipping downlink throughput during the header->filter transition. The + // extend is a no-op once the ceiling has reached the tip. + let ceiling = self.download_ceiling().await; + self.filter_pipeline.extend_target(ceiling); + + // Timeouts/retry are the network manager's job now (the broker re-injects); + // just push pending (decoupled from processing). let header_storage = self.header_storage.read().await; self.filter_pipeline.send_pending(network, &*header_storage).await?; drop(header_storage); diff --git a/dash-spv/tests/dashd_sync/tests_multi_wallet.rs b/dash-spv/tests/dashd_sync/tests_multi_wallet.rs index 2dc20d57b..57af96d27 100644 --- a/dash-spv/tests/dashd_sync/tests_multi_wallet.rs +++ b/dash-spv/tests/dashd_sync/tests_multi_wallet.rs @@ -389,18 +389,24 @@ async fn test_runtime_add_during_initial_sync() { let mut client_handle = create_and_start_client(&ctx.client_config, Arc::clone(&wallet)).await; - let midpoint = initial_height / 2; + // Add W2 at the FIRST sign of mid-flight progress (any 0 < committed < tip), + // not at a fixed midpoint. `committed_height` advances in coarse + // BATCH_PROCESSING_SIZE steps, and `progress_receiver` is a `watch` that only + // keeps the latest value: when filter sync is fast the producer can emit several + // steps between two reads here, so the consumer may jump straight past a midpoint + // window to the tip. Breaking on the earliest committed step makes the catchable + // window as wide as possible (any step below the tip qualifies) while still + // guaranteeing W1 is genuinely mid-flight when W2 is added. let inflight_deadline = tokio::time::Instant::now() + Duration::from_secs(60); loop { let filter_height = { let progress = client_handle.progress_receiver.borrow_and_update(); progress.filters().ok().map(|f| f.committed_height()).unwrap_or(0) }; - if filter_height >= midpoint && filter_height < initial_height { + if filter_height > 0 && filter_height < initial_height { tracing::info!( - "Mid-flight: filter committed_height={} (midpoint={}, tip={})", + "Mid-flight: filter committed_height={} (tip={})", filter_height, - midpoint, initial_height, ); break; @@ -408,14 +414,14 @@ async fn test_runtime_add_during_initial_sync() { if filter_height >= initial_height { panic!( "filter sync reached tip ({}) before W2 could be added; \ - lower the midpoint threshold or use a larger TestChain", + use a larger TestChain so mid-flight lasts longer", initial_height, ); } if tokio::time::Instant::now() > inflight_deadline { panic!( - "filter committed_height did not reach midpoint {} within 60s, stuck at {}", - midpoint, filter_height, + "filter committed_height did not advance past 0 within 60s, stuck at {}", + filter_height, ); } tokio::select! { From 210243c863ac2dfae37116b654ee3cae42625627 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Fri, 17 Jul 2026 13:38:09 +0000 Subject: [PATCH 20/25] hope this works --- dash-spv/src/network/manager.rs | 3 +- dash-spv/src/sync/block_headers/manager.rs | 18 +++++-- dash-spv/src/sync/events.rs | 9 ++++ .../src/sync/filter_headers/sync_manager.rs | 1 + dash-spv/src/sync/filters/manager.rs | 53 +++++++++++++++++++ dash-spv/src/sync/filters/pipeline.rs | 27 ++++++++++ dash-spv/src/sync/filters/progress.rs | 19 ++++--- dash-spv/src/sync/filters/sync_manager.rs | 32 +++++++++-- 8 files changed, 145 insertions(+), 17 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 5896511c1..56b766d85 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -913,7 +913,7 @@ fn spawn_bandwidth_controller( tracing::debug!( target: "peer_speed", - "peer {}: cap={} in_flight={} lag={}ms lambda={:.1}/s W={:.0}ms rate={:.2} MB/s", + "peer {}: cap={} in_flight={} lag={}ms lambda={:.1}/s W={:.0}ms rate={:.2} MB/s total={:.1} MB", addr, cap_peer, p.in_flight(), @@ -921,6 +921,7 @@ fn spawn_bandwidth_controller( lambda, w * 1e3, rate / 1e6, + p.bytes_read() as f64 / 1e6, ); cap_min = cap_min.min(cap_peer); diff --git a/dash-spv/src/sync/block_headers/manager.rs b/dash-spv/src/sync/block_headers/manager.rs index c69f378bd..11472c1d8 100644 --- a/dash-spv/src/sync/block_headers/manager.rs +++ b/dash-spv/src/sync/block_headers/manager.rs @@ -138,8 +138,9 @@ impl BlockHeadersManager { // cost (~70ms per 8000-header batch), and it otherwise runs serially on this // single manager task. Split into 4 chunks across tokio's runtime workers, then // feed the pre-hashed headers to the pipeline (validates checkpoint match). - let (matched, answered) = if headers.is_empty() { - self.pipeline.receive_headers_prehashed(&[])? + let (matched, answered, batch_hashes) = if headers.is_empty() { + let (m, a) = self.pipeline.receive_headers_prehashed(&[])?; + (m, a, Vec::new()) } else { let len = headers.len(); let chunk = len.div_ceil(4).max(1); @@ -161,7 +162,12 @@ impl BlockHeadersManager { } parts.sort_by_key(|(i, _)| *i); let hashed: Vec = parts.into_iter().flat_map(|(_, v)| v).collect(); - self.pipeline.receive_headers_prehashed(&hashed)? + // The accepted range is a contiguous prefix of `hashed` starting at the + // segment's next height, so these hashes line up with the announced + // `start_height..=end_height` below (sliced to the accepted length). + let hashes: Vec = hashed.iter().map(|h| *h.hash()).collect(); + let (m, a) = self.pipeline.receive_headers_prehashed(&hashed)?; + (m, a, hashes) }; // Correlated to an outstanding GetHeaders: stop the network manager @@ -192,10 +198,16 @@ impl BlockHeadersManager { // parallel without waiting for the sequential store head. let mut events = Vec::new(); if let Some((_, start_height, end_height, stop_hash)) = matched { + // `batch_hashes` covers what this response accepted; the announced range + // is that same accepted prefix, so take exactly `[start..=end]`. + let n = (end_height - start_height + 1) as usize; + let mut hashes = batch_hashes; + hashes.truncate(n); events.push(SyncEvent::BlockHeadersInMemory { start_height, end_height, stop_hash, + hashes: Arc::new(hashes), }); } diff --git a/dash-spv/src/sync/events.rs b/dash-spv/src/sync/events.rs index 6d627edf7..9b67ba458 100644 --- a/dash-spv/src/sync/events.rs +++ b/dash-spv/src/sync/events.rs @@ -45,6 +45,15 @@ pub enum SyncEvent { end_height: u32, /// Block hash at `end_height` (the `getcfheaders` stop hash). stop_hash: BlockHash, + /// The block hashes for `start_height..=end_height`, in height order. + /// + /// Carried so `FiltersManager` can drive `getcfilters` off the in-memory + /// frontier without waiting for the in-order block-header store: a + /// `getcfilters` batch needs the block hash at its (1000-aligned) stop + /// height, and cfilter responses need a `hash -> height` lookup that + /// storage can't yet answer for a segment validated but not stored. An + /// `Arc` so the batch (≤2000 hashes) is not cloned per subscriber. + hashes: Arc>, }, /// Headers have reached the chain tip (initial sync complete). diff --git a/dash-spv/src/sync/filter_headers/sync_manager.rs b/dash-spv/src/sync/filter_headers/sync_manager.rs index f4fd03600..bd135452e 100644 --- a/dash-spv/src/sync/filter_headers/sync_manager.rs +++ b/dash-spv/src/sync/filter_headers/sync_manager.rs @@ -165,6 +165,7 @@ impl SyncManager for FilterHeade start_height, end_height, stop_hash, + .. } => { self.handle_headers_in_memory(*start_height, *end_height, *stop_hash, network).await } diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index 05d647c0e..6ebd43858 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -506,6 +506,14 @@ pub struct FiltersManager< /// reach, so they arrive around the same time as these do — keeping them here /// means the verification never has to read back what we were just handed. pub(super) filter_headers: BTreeMap, + /// Block `hash -> height` for headers validated in memory (via + /// `BlockHeadersInMemory`) but not yet written to `header_storage` by the + /// in-order store. Lets a `cfilter` requested off the in-memory frontier be + /// attributed to its height even though storage can't answer yet. Pruned each + /// tick once the in-order store has caught up past those heights, so it holds + /// only the in-memory lead (a handful of out-of-order segments), never the + /// whole chain. + pub(super) mem_block_heights: HashMap, /// Cached wallet birth (`earliest_required_height`), refreshed each /// `start_download`. Lets the BlockProcessed handler roll a wallet's frontier /// back to birth WITHOUT taking a wallet lock — doing that read inside the @@ -571,6 +579,7 @@ impl height` so cfilters we + /// request now can be attributed even though storage can't answer yet, extends + /// the download target to the in-memory frontier, and declares the newly + /// unlocked batches to the broker. A no-op unless filters are actively syncing: + /// before `start_download` the pipeline has no batches to unlock, and the + /// eventual start reads the (by then stored) headers from storage. + pub(super) async fn handle_block_headers_in_memory( + &mut self, + start_height: u32, + end_height: u32, + hashes: &[BlockHash], + network: &Arc, + ) -> SyncResult> { + if self.state() != SyncState::Syncing { + return Ok(vec![]); + } + for (i, hash) in hashes.iter().enumerate() { + self.mem_block_heights.insert(*hash, start_height + i as u32); + } + self.filter_pipeline.extend_target(end_height); + self.filter_pipeline.seed_stops_from_memory(start_height, hashes); + let header_storage = self.header_storage.read().await; + self.filter_pipeline.send_pending(network, &*header_storage).await?; + Ok(vec![]) + } + + /// Drop `hash -> height` entries the in-order store has caught up past: once a + /// block header sits at or below the stored tip, `header_storage` can resolve it, + /// so the in-memory fallback is no longer needed. Keeps the map bounded to the + /// in-memory lead rather than the whole chain. + pub(super) async fn prune_mem_block_heights(&mut self) { + if self.mem_block_heights.is_empty() { + return; + } + let stored_tip = + self.header_storage.read().await.get_tip().await.map(|t| t.height()).unwrap_or(0); + self.mem_block_heights.retain(|_, &mut height| height > stored_tip); + } } impl diff --git a/dash-spv/src/sync/filters/pipeline.rs b/dash-spv/src/sync/filters/pipeline.rs index 3404d1fca..148a38b1d 100644 --- a/dash-spv/src/sync/filters/pipeline.rs +++ b/dash-spv/src/sync/filters/pipeline.rs @@ -182,6 +182,33 @@ impl FiltersPipeline { Ok(n) } + /// Seed stop hashes for wanted batches from block hashes already validated in + /// memory (a `BlockHeadersInMemory` segment), so the next `send_pending` can + /// declare those batches without waiting for the in-order block-header store. + /// + /// `hashes` covers `start_height..` in height order. Only wanted batches whose + /// stop height (`batch_end`) falls inside that range — and aren't already + /// cached — get seeded. + pub(super) fn seed_stops_from_memory(&mut self, start_height: u32, hashes: &[BlockHash]) { + if hashes.is_empty() { + return; + } + let end_height = start_height + hashes.len() as u32 - 1; + let to_seed: Vec<(u32, BlockHash)> = self + .batch_trackers + .iter() + .filter(|(start, _)| !self.batch_stops.contains_key(start)) + .filter_map(|(&start, tracker)| { + let batch_end = tracker.end_height(); + (batch_end >= start_height && batch_end <= end_height) + .then(|| (start, hashes[(batch_end - start_height) as usize])) + }) + .collect(); + for (start, stop) in to_seed { + self.batch_stops.insert(start, stop); + } + } + /// Handle a received CFilter message with filter data. /// /// Buffers the filter data for batch verification and wallet matching. diff --git a/dash-spv/src/sync/filters/progress.rs b/dash-spv/src/sync/filters/progress.rs index 10472e7f0..d9e30de96 100644 --- a/dash-spv/src/sync/filters/progress.rs +++ b/dash-spv/src/sync/filters/progress.rs @@ -148,12 +148,12 @@ impl fmt::Display for FiltersProgress { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "{:?} {}/{} ({:.1}%) committed: {}, downloaded: {}, processed: {}, matched: {}, last_activity: {}s", + "{:?} {}/{} ({:.1}%) stored: {}, downloaded: {}, processed: {}, matched: {}, last_activity: {}s", self.state, self.current_height(), self.target_height(), self.percentage() * 100.0, - self.committed_height, + self.stored_height, self.downloaded, self.processed, self.matched, @@ -166,15 +166,14 @@ impl ProgressPercentage for FiltersProgress { fn target_height(&self) -> u32 { self.target_height } - /// How far the FILTERS themselves are down: the height up to which they have - /// been downloaded, verified and stored. + /// How far filter PROCESSING has reached: the committed/scan frontier, i.e. the + /// height up to which every filter has been matched and its blocks fully + /// handled. This is the real work-done metric. /// - /// Not `committed_height`, which is the scan/commit frontier — that trails the - /// download by a long way (it waits on the matched blocks and the gap-limit - /// rescan), so reporting it made the filters look stalled while they were in - /// fact streaming in, and hid the fact that they download in parallel with the - /// filter headers. `committed_height` is still reported in the detail fields. + /// The download/store frontier (`stored_height`) runs well ahead of this — it is + /// reported in the `stored:` detail field — because processing waits on the + /// matched blocks and the gap-limit rescan behind the filters that stream in. fn current_height(&self) -> u32 { - self.stored_height + self.committed_height } } diff --git a/dash-spv/src/sync/filters/sync_manager.rs b/dash-spv/src/sync/filters/sync_manager.rs index 7e796c741..770d9ea9b 100644 --- a/dash-spv/src/sync/filters/sync_manager.rs +++ b/dash-spv/src/sync/filters/sync_manager.rs @@ -112,9 +112,19 @@ impl< return Ok(vec![]); }; - // Find height for this filter - let height = - self.header_storage.read().await.get_header_height_by_hash(&cfilter.block_hash).await?; + // Find height for this filter. Storage first; fall back to the in-memory + // block-header map for a cfilter we requested off the in-memory frontier + // whose header the in-order store has not written yet. + let height = match self + .header_storage + .read() + .await + .get_header_height_by_hash(&cfilter.block_hash) + .await? + { + Some(h) => Some(h), + None => self.mem_block_heights.get(&cfilter.block_hash).copied(), + }; let Some(h) = height else { tracing::warn!( @@ -162,6 +172,19 @@ impl< return self.handle_new_filter_headers(*tip_height, network).await; } + // Block headers validated in memory (before the in-order store): drive + // getcfilters off that frontier instead of waiting for the store. + SyncEvent::BlockHeadersInMemory { + start_height, + end_height, + hashes, + .. + } => { + return self + .handle_block_headers_in_memory(*start_height, *end_height, hashes, network) + .await; + } + SyncEvent::FilterHeadersStored { start_height, tip_height, @@ -243,6 +266,9 @@ impl< // Bounds staleness to one tick; a hard kill in that window just rescans. self.persist_scan_state().await; + // Drop in-memory block-header entries the in-order store has caught up past. + self.prune_mem_block_heights().await; + // Detect a wallet that was added behind our scan progress and rescan // from its `synced_height`. Reset committed_height to the lowest // synced_height across the stale wallets only, so already-synced From e69cbd7f14b102f88db5243c10202f3ea2d7d4e6 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Fri, 17 Jul 2026 13:57:25 +0000 Subject: [PATCH 21/25] drop the task that connected th cliento to more peers udner pressure --- dash-spv/src/network/manager.rs | 59 +++------------------------------ 1 file changed, 4 insertions(+), 55 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 56b766d85..bd994e629 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -17,20 +17,8 @@ use tokio::sync::{broadcast, Mutex, Notify}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; -const HIGH_DEMAND_QUEUE: usize = 40; const ADD_PEERS_BATCH: usize = 4; const PROBE_BATCH: usize = 256; -/// Safety ceiling on how many peers backpressure may auto-connect. `max_peers` -/// (config) is the base/initial set; when the send queue stays backed up the -/// peers can't serve fast enough, so we recruit more — up to this cap — to raise -/// aggregate serving capacity. -/// -/// Kept near the empirical sweet spot: on testnet, throughput improves up to -/// ~16 peers, is flat-to-worse by 24-32, and OUTRIGHT STALLS around 64 — dozens -/// of slow peers each pin their measured in-flight on batches that never finish, -/// so there are no completions to wake the router. More peers past the knee only -/// adds congestion, not download rate (the peers, not our link, are the limit). -const MAX_CONNECTED_PEERS: usize = 16; /// How often the reconnector checks whether the peer set needs topping up. const RECONNECT_CHECK: Duration = Duration::from_secs(2); @@ -310,15 +298,9 @@ impl PeerNetworkManager { spawn_router( msg_queue.clone(), connected_peers.clone(), - other_peers.clone(), - inbound_tx.clone(), shutdown.clone(), - bytes.clone(), global_cap.clone(), - events_tx.clone(), - best_tip.clone(), requests.clone(), - required_services, ); spawn_timeout_monitor( @@ -527,19 +509,12 @@ impl PeerNetworkManager { } } -#[allow(clippy::too_many_arguments)] fn spawn_router( queue: Arc, connected: Arc>>, - others: Arc>>, - inbound: UnboundedSender, shutdown: CancellationToken, - bytes: Arc, global_cap: Arc, - events: broadcast::Sender, - best_tip: Arc, requests: Registry, - required_services: ServiceFlags, ) -> JoinHandle<()> { tokio::spawn(async move { loop { @@ -555,31 +530,6 @@ fn spawn_router( } } - // Auto-connect more peers while the send queue is backed up: a deep - // queue means the connected peers can't serve our requests as fast as - // the pipelines produce them, so recruiting more raises the aggregate - // serving capacity and drains it — pushing the bottleneck toward OUR - // download link. Growth is bounded by `MAX_CONNECTED_PEERS`. Safe to - // grow past the initial `max_peers` now that each peer self-throttles - // to its MEASURED cap (a slow newcomer gets a tiny cap instead of - // feeding the old over-commit spiral). - if queue.len() > HIGH_DEMAND_QUEUE && connected.lock().await.len() < MAX_CONNECTED_PEERS - { - add_peers( - &connected, - &others, - &inbound, - &shutdown, - &bytes, - &events, - &best_tip, - ADD_PEERS_BATCH, - MAX_CONNECTED_PEERS, - required_services, - ) - .await; - } - let peers = connected.lock().await; let sent = route_tick(&queue, &peers, global_cap.load(Ordering::Relaxed), &requests).await; @@ -971,11 +921,10 @@ fn spawn_bandwidth_controller( /// Keep the peer set topped up. /// -/// Peers are connected once, in `start`; nothing put them back afterwards. The router's -/// `add_peers` only fires under backpressure (a deep send queue), so a client whose peers -/// all dropped while it was idle — or mid-sync with a short queue — simply sat there with -/// zero peers forever. This watches the count and refills it, pulling fresh candidates -/// from the discoverer when the backup list runs dry. +/// Peers are connected once, in `start`; nothing put them back afterwards, so a client +/// whose peers all dropped — while idle or mid-sync — would simply sit there with zero +/// peers forever. This watches the count and refills it back to `max_peers`, pulling +/// fresh candidates from the discoverer when the backup list runs dry. #[allow(clippy::too_many_arguments)] fn spawn_reconnector( discoverer: Arc>, From d9f944080538f8dc2415b498c175795c42051e79 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Fri, 17 Jul 2026 15:00:37 +0000 Subject: [PATCH 22/25] requested more servivees from the peers --- dash-spv/src/network/manager.rs | 18 +++++++++++++----- dash-spv/src/network/peer.rs | 10 ++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index bd994e629..5d7365073 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -256,13 +256,21 @@ impl PeerNetworkManager { pub async fn new(config: &ClientConfig) -> Self { let discoverer = Arc::new(Mutex::new(PeerDiscoverer::new(config))); let max_peers = config.max_peers.max(1) as usize; + // NETWORK is unconditional: the block pipeline asks for full blocks at + // arbitrary historical heights (and the gap-limit rescan far more so), which + // a NETWORK_LIMITED peer stops serving past its last ~288 blocks. + let mut required_services = ServiceFlags::NETWORK; // A filter-syncing client can only use peers that serve compact filters // (BIP157 — the same flag also covers compact filter headers). - let required_services = if config.enable_filters { - ServiceFlags::COMPACT_FILTERS - } else { - ServiceFlags::NONE - }; + if config.enable_filters { + required_services |= ServiceFlags::COMPACT_FILTERS; + } + // Mempool tracking sends `mempool` to every activated peer, and `filterload` + // too under the BloomFilter strategy. A peer with bloom filters disabled + // answers the first by dropping us and the second, per BIP111, by banning us. + if config.enable_mempool_tracking { + required_services |= ServiceFlags::BLOOM; + } let connected_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); let other_peers = Arc::new(Mutex::new(Vec::with_capacity(30))); diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 2967a9b16..6c9d229d4 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -345,12 +345,10 @@ impl DisconnectedPeer { let version = peer_version.ok_or(NetworkError::PeerDisconnected)?; - // Only keep peers that advertise the services we need. For a filter-syncing - // client that means COMPACT_FILTERS (BIP157) — the SAME flag gates cfilters - // AND cfheaders. A peer without it serves headers/blocks fine but silently - // ignores `getcfilters`, so its slots would stall until the request times - // out; drop it now rather than waste it. `has(NONE)` is always true, so an - // empty requirement (filters disabled) keeps every peer. + // Only keep peers that advertise the services we need (see `new` in `manager` + // for how the set is composed). A peer missing one of them doesn't fail loudly: + // it stays connected and simply ignores the requests it can't serve, so its + // slots stall until the request times out. Drop it now rather than waste it. if !version.services.has(required_services) { tracing::debug!( target: "dash_spv::network", From 6a86b193e451184ae1bce4a60677471ec4a608c1 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Fri, 17 Jul 2026 18:14:25 +0000 Subject: [PATCH 23/25] peer hot swap prototype --- dash-spv/src/client/lifecycle.rs | 2 +- dash-spv/src/network/manager.rs | 528 +++++++++++++++++-------------- dash-spv/src/network/peer.rs | 14 + 3 files changed, 301 insertions(+), 243 deletions(-) diff --git a/dash-spv/src/client/lifecycle.rs b/dash-spv/src/client/lifecycle.rs index 797c547d7..5b53f6ec5 100644 --- a/dash-spv/src/client/lifecycle.rs +++ b/dash-spv/src/client/lifecycle.rs @@ -202,7 +202,7 @@ impl DashSpvClient { return Err(SpvError::Sync(e)); } - self.network.start().await; + self.network.start(); // Only mark as running after all startup operations succeed — and only if // no `stop()` raced in while we were connecting. The check runs inside the diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 5d7365073..18a7f831f 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -17,11 +17,33 @@ use tokio::sync::{broadcast, Mutex, Notify}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; -const ADD_PEERS_BATCH: usize = 4; -const PROBE_BATCH: usize = 256; +/// Bounded concurrent handshakes per probe round. +const CONNECT_CHUNK: usize = 16; -/// How often the reconnector checks whether the peer set needs topping up. -const RECONNECT_CHECK: Duration = Duration::from_secs(2); +/// Candidates probed per improve round (at cap). Small to bound connect/close churn +/// while still discovering a better peer over time. +const IMPROVE_PROBE: usize = 4; + +/// Handshake ping below which a peer is "decent": preferred when filling, and the +/// bar a candidate must clear to be allowed to displace a slow connected peer. +const DECENT_LAG_MS: u32 = 100; + +/// Handshake ping at/above which a peer is "very bad": taken only as a last resort, +/// when nothing better is connectable and the set would otherwise be empty. +const BAD_LAG_MS: u32 = 1000; + +/// A candidate displaces the worst connected peer only if its ping is at most this +/// fraction of the worst peer's — i.e. clearly, not marginally, better. +const SWAP_IMPROVEMENT: u32 = 2; + +/// How often the supervisor re-checks a below-cap set and probes to keep filling. +const FILL_TICK: Duration = Duration::from_secs(2); + +/// How often the supervisor probes for a better peer once the set is at capacity. +const IMPROVE_TICK: Duration = Duration::from_secs(5); + +/// Cap on remembered (ranked) backup addresses, as a multiple of `max_peers`. +const BACKUP_MULTIPLE: usize = 8; /// How long the router sleeps on a full-capacity stall before re-evaluating. It /// wakes early whenever a response frees a slot or the timeout monitor kicks a @@ -325,18 +347,8 @@ impl PeerNetworkManager { shutdown.clone(), ); - spawn_reconnector( - discoverer.clone(), - connected_peers.clone(), - other_peers.clone(), - inbound_tx.clone(), - shutdown.clone(), - bytes.clone(), - events_tx.clone(), - best_tip.clone(), - max_peers, - required_services, - ); + // The peer supervisor is spawned by `start()`, not here: it must not emit + // `PeersUpdated` until the sync managers have subscribed (see `start`). PeerNetworkManager { connected_peers, @@ -363,35 +375,26 @@ impl PeerNetworkManager { /// send `filterload` to enable transaction relay — ended up with an empty set and never /// activated. Build the manager, let the coordinator spawn and subscribe its managers, /// then call this. - pub async fn start(&self) { - let best_tip = probe_and_select( - &mut *self.discoverer.lock().await, - &self.connected_peers, - &self.other_peers, + pub fn start(&self) { + // Non-blocking: spawn the supervisor and return. It probes peers, connects + // the decent ones, and emits `PeerConnected`/`PeersUpdated` as they arrive — + // so sync begins the moment the first decent peer is up, without `start` + // waiting on peer discovery. Spawned here rather than in `new` so it runs + // only after the coordinator has subscribed its managers; otherwise the first + // `PeersUpdated` (and the mempool's `filterload` trigger) would fire into the + // void. + spawn_peer_supervisor( + self.discoverer.clone(), + self.connected_peers.clone(), + self.other_peers.clone(), + self.inbound_tx.clone(), + self.shutdown.clone(), + self.bytes.clone(), + self.events_tx.clone(), + self.best_tip.clone(), self.max_peers, - &self.inbound_tx, - &self.shutdown, - &self.bytes, self.required_services, - ) - .await; - - self.best_tip.fetch_max(best_tip, Ordering::Relaxed); - - // Announce each peer individually before the summary. Managers that track the peer - // set (the mempool, which must send `filterload` to turn on transaction relay) build - // it from these; `PeersUpdated` alone carries no addresses. - let addrs: Vec = - self.connected_peers.lock().await.iter().map(|(peer, _)| peer.addr()).collect(); - let connected_count = addrs.len() as u32; - for addr in addrs { - let _ = self.events_tx.send(NetworkEvent::PeerConnected(addr)); - } - - let _ = self.events_tx.send(NetworkEvent::PeersUpdated { - connected_count, - best_height: best_tip, - }); + ); } /// Tear down the network layer: stop the router and pump, and cancel every @@ -934,7 +937,35 @@ fn spawn_bandwidth_controller( /// peers forever. This watches the count and refills it back to `max_peers`, pulling /// fresh candidates from the discoverer when the backup list runs dry. #[allow(clippy::too_many_arguments)] -fn spawn_reconnector( +/// Sort key for a connected peer's handshake ping: lower is better, and an +/// unmeasured lag (0) sorts as worst. +fn lag_key(peer: &ConnectedPeer) -> u32 { + match peer.lag_ms() { + 0 => u32::MAX, + ms => ms, + } +} + +/// The peer supervisor: the single task that owns the connected-peer set. +/// +/// It keeps `connected` filled toward `max_peers` with the lowest-latency peers it +/// can find, without ever blocking the caller: +/// +/// - Below cap it probes candidates in parallel and accepts the decent ones +/// (handshake ping under [`BAD_LAG_MS`]), emitting `PeersUpdated` as they connect +/// so sync starts on the first one. A "very bad" peer is taken only as a last +/// resort — when the set would otherwise be empty and nothing better connected. +/// - At cap it probes a few candidates every [`IMPROVE_TICK`] and, if one is clearly +/// better (ping ≤ worst / [`SWAP_IMPROVEMENT`]) than the slowest connected peer, +/// swaps it in. The displaced peer is handed to [`retire_drained`] so its in-flight +/// requests finish (or time out) before its socket closes. +/// - A peer kicked by the timeout monitor just drops the set below cap, so the next +/// fill round refills it — the same path as any other deficit. +/// +/// Probed-but-unused peers are closed and kept as ranked backups (`others`, carrying +/// their measured ping) so a later round reconnects the best of them without probing +/// blindly. +struct Supervisor { discoverer: Arc>, connected: Arc>>, others: Arc>>, @@ -945,71 +976,230 @@ fn spawn_reconnector( best_tip: Arc, max_peers: usize, required_services: ServiceFlags, -) -> JoinHandle<()> { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(RECONNECT_CHECK); +} +impl Supervisor { + async fn run(self) { loop { + let at_cap = self.connected.lock().await.len() >= self.max_peers; + let dur = if at_cap { + self.improve_round().await; + IMPROVE_TICK + } else { + self.fill_round().await; + FILL_TICK + }; tokio::select! { - _ = shutdown.cancelled() => break, - _ = ticker.tick() => {} + _ = self.shutdown.cancelled() => break, + _ = tokio::time::sleep(dur) => {} } + } + } - let deficit = max_peers.saturating_sub(connected.lock().await.len()); - if deficit == 0 { - continue; + /// Up to `want` candidate addresses to probe: best-ranked backups first, then + /// fresh discovery, de-duplicated against each other and the live set. + async fn next_candidates(&self, want: usize) -> Vec { + let mut out: Vec = { + let mut o = self.others.lock().await; + o.sort_by_key(|p| p.lag_ms().unwrap_or(u32::MAX)); + let take = want.min(o.len()); + o.drain(..take).collect() + }; + if out.len() < want { + out.extend(self.discoverer.lock().await.get(want - out.len()).await); + } + let live: HashSet = + self.connected.lock().await.iter().map(|(p, _)| p.addr()).collect(); + let mut seen = HashSet::new(); + out.retain(|p| !live.contains(&p.addr()) && seen.insert(p.addr())); + out + } + + /// Connect a batch in parallel, keeping the successful handshakes and advancing + /// `best_tip` from whatever chain height they advertise. + async fn connect_chunk(&self, batch: Vec) -> Vec { + let results = join_all(batch.into_iter().map(|c| { + c.connect( + self.inbound.clone(), + self.shutdown.clone(), + self.bytes.clone(), + self.required_services, + ) + })) + .await; + let peers: Vec = results.into_iter().filter_map(Result::ok).collect(); + for p in &peers { + self.best_tip.fetch_max(p.version().start_height.max(0) as u32, Ordering::Relaxed); + } + peers + } + + /// Close probed-but-unused peers and remember them as ranked backups, keeping the + /// list de-duplicated (best ping per address) and bounded. + async fn stash_backups(&self, peers: impl IntoIterator) { + let mut o = self.others.lock().await; + for peer in peers { + peer.close(); + o.push(peer.disconnect()); + } + // Dedup by address keeping the lowest ping, then rank and cap the list. + o.sort_by(|a, b| { + a.addr() + .cmp(&b.addr()) + .then(a.lag_ms().unwrap_or(u32::MAX).cmp(&b.lag_ms().unwrap_or(u32::MAX))) + }); + o.dedup_by_key(|p| p.addr()); + o.sort_by_key(|p| p.lag_ms().unwrap_or(u32::MAX)); + o.truncate(self.max_peers * BACKUP_MULTIPLE); + } + + async fn announce_update(&self) { + let count = self.connected.lock().await.len() as u32; + let _ = self.events.send(NetworkEvent::PeersUpdated { + connected_count: count, + best_height: self.best_tip.load(Ordering::Relaxed), + }); + } + + /// Below cap: probe and accept decent peers, emitting `PeersUpdated` as they land. + async fn fill_round(&self) { + if self.max_peers.saturating_sub(self.connected.lock().await.len()) == 0 { + return; + } + let batch = self.next_candidates(CONNECT_CHUNK).await; + if batch.is_empty() { + return; + } + let mut probed = self.connect_chunk(batch).await; + probed.sort_by_key(lag_key); // best first + + let mut accepted = 0usize; + let mut leftover: Vec = Vec::new(); + for peer in probed { + let lag = peer.lag_ms(); + let acceptable = lag > 0 && lag < BAD_LAG_MS; + if acceptable && self.connected.lock().await.len() < self.max_peers { + let addr = peer.addr(); + self.connected.lock().await.push((peer, State {})); + let _ = self.events.send(NetworkEvent::PeerConnected(addr)); + accepted += 1; + } else { + leftover.push(peer); } + } - // Backups first (already discovered, not yet used); otherwise go ask for more. - if others.lock().await.is_empty() { - let fresh = discoverer.lock().await.get(deficit.max(ADD_PEERS_BATCH)).await; + // Last resort: never sit at zero peers. If nothing decent connected and the + // set is empty, take the least-bad handshake we got so sync can start; the + // improve loop upgrades it once a decent peer appears. + if accepted == 0 && self.connected.lock().await.is_empty() && !leftover.is_empty() { + leftover.sort_by_key(lag_key); + let peer = leftover.remove(0); + let addr = peer.addr(); + tracing::warn!( + target: "dash_spv::network", + "no decent peer available; accepting {} (ping {}ms) as last resort", + addr, + peer.lag_ms(), + ); + self.connected.lock().await.push((peer, State {})); + let _ = self.events.send(NetworkEvent::PeerConnected(addr)); + accepted += 1; + } - // The discoverer hands out of a pool it keeps, so it can name a peer we are - // already talking to. Filter those out rather than opening a second socket - // to the same node. - let live: HashSet = - connected.lock().await.iter().map(|(peer, _)| peer.addr()).collect(); - let fresh: Vec = - fresh.into_iter().filter(|p| !live.contains(&p.addr())).collect(); + self.stash_backups(leftover).await; + + if accepted > 0 { + self.announce_update().await; + tracing::info!( + target: "dash_spv::network", + "peer supervisor: +{} peers -> {}", + accepted, + self.connected.lock().await.len(), + ); + } + } - if fresh.is_empty() { - continue; + /// At cap: probe a few candidates and swap the slowest connected peer for a + /// clearly-faster one, retiring the displaced peer so its in-flight work drains. + async fn improve_round(&self) { + let batch = self.next_candidates(IMPROVE_PROBE).await; + if batch.is_empty() { + return; + } + let mut probed = self.connect_chunk(batch).await; + probed.sort_by_key(lag_key); // best first + + let mut swapped: Option<(SocketAddr, ConnectedPeer)> = None; + if let Some(cand_lag) = probed.first().map(ConnectedPeer::lag_ms) { + if cand_lag > 0 && cand_lag < DECENT_LAG_MS { + let mut peers = self.connected.lock().await; + if let Some((pos, worst_lag)) = peers + .iter() + .enumerate() + .map(|(i, (p, _))| (i, lag_key(p))) + .max_by_key(|&(_, l)| l) + { + let clearly_better = worst_lag > DECENT_LAG_MS + && cand_lag.saturating_mul(SWAP_IMPROVEMENT) <= worst_lag; + if clearly_better && peers.len() >= self.max_peers { + let candidate = probed.remove(0); + let new_addr = candidate.addr(); + let (old, _) = peers.swap_remove(pos); + peers.push((candidate, State {})); + swapped = Some((new_addr, old)); + } } - others.lock().await.extend(fresh); } + } - let before = connected.lock().await.len(); - add_peers( - &connected, - &others, - &inbound, - &shutdown, - &bytes, - &events, - &best_tip, - deficit, - max_peers, - required_services, - ) - .await; - let after = connected.lock().await.len(); - - if after > before { - tracing::info!( - target: "dash_spv::network", - "reconnected: {} -> {} peers", - before, - after, - ); - // Wake the managers: those that stopped for lack of peers restart, and the - // mempool re-arms relay on the new peers. - let _ = events.send(NetworkEvent::PeersUpdated { - connected_count: after as u32, - best_height: best_tip.load(Ordering::Relaxed), - }); - } + if let Some((new_addr, old)) = swapped { + let old_addr = old.addr(); + tracing::info!( + target: "dash_spv::network", + "peer supervisor: swapped out slow {} for faster {}", + old_addr, + new_addr, + ); + // Keep the displaced peer alive until its in-flight requests drain or time + // out, then close it — don't strand work already routed to it. + retire_drained(old, self.shutdown.clone()); + let _ = self.events.send(NetworkEvent::PeerConnected(new_addr)); + let _ = self.events.send(NetworkEvent::PeerDisconnected(old_addr)); + self.announce_update().await; } - }) + + self.stash_backups(probed).await; + } +} + +#[allow(clippy::too_many_arguments)] +fn spawn_peer_supervisor( + discoverer: Arc>, + connected: Arc>>, + others: Arc>>, + inbound: UnboundedSender, + shutdown: CancellationToken, + bytes: Arc, + events: broadcast::Sender, + best_tip: Arc, + max_peers: usize, + required_services: ServiceFlags, +) -> JoinHandle<()> { + tokio::spawn( + Supervisor { + discoverer, + connected, + others, + inbound, + shutdown, + bytes, + events, + best_tip, + max_peers, + required_services, + } + .run(), + ) } /// Time requests out and evict the peers that stalled on them. @@ -1139,59 +1329,6 @@ fn spawn_timeout_monitor( }) } -#[allow(clippy::too_many_arguments)] -async fn add_peers( - connected: &Mutex>, - others: &Mutex>, - inbound: &UnboundedSender, - shutdown: &CancellationToken, - bytes: &Arc, - events: &broadcast::Sender, - best_tip: &Arc, - batch: usize, - max_peers: usize, - required_services: ServiceFlags, -) { - // Never grow past the operating cap; only backfill the deficit toward it. - let deficit = max_peers.saturating_sub(connected.lock().await.len()); - if deficit == 0 { - return; - } - let candidates: Vec = { - let mut others = others.lock().await; - let take = batch.min(deficit).min(others.len()); - others.drain(..take).collect() - }; - - let mut added = 0; - for candidate in candidates { - if let Ok(peer) = candidate - .connect(inbound.clone(), shutdown.clone(), bytes.clone(), required_services) - .await - { - let addr = peer.addr(); - // Keep `best_tip` current the moment a peer arrives: a newcomer may - // advertise a higher chain tip than the initial probe saw, and the sync - // managers seed their target height from `best_tip` on `PeersUpdated`, so - // it must be right before that event fires — not only after the probe. - best_tip.fetch_max(peer.version().start_height.max(0) as u32, Ordering::Relaxed); - connected.lock().await.push((peer, State {})); - let _ = events.send(NetworkEvent::PeerConnected(addr)); - added += 1; - } - } - - if added > 0 { - let total = connected.lock().await.len(); - tracing::info!( - target: "dash_spv::network", - "high demand: added {} peers -> {} connected", - added, - total, - ); - } -} - /// Retire a peer we are dropping from the active set WITHOUT stranding requests /// already on the wire to it. During startup the reconnector connects peers and /// the sync sends them pipeline requests before the probe has settled the final @@ -1218,99 +1355,6 @@ fn retire_drained(peer: ConnectedPeer, shutdown: CancellationToken) { }); } -#[allow(clippy::too_many_arguments)] -async fn probe_and_select( - discoverer: &mut PeerDiscoverer, - connected: &Mutex>, - others: &Mutex>, - max_peers: usize, - inbound: &UnboundedSender, - shutdown: &CancellationToken, - bytes: &Arc, - required_services: ServiceFlags, -) -> u32 { - const CONNECT_CHUNK: usize = 16; // bounded concurrent handshakes per round - const FAST_LAG_MS: u32 = 100; // prefer peers that ping under this - - // Sort key for latency: lower is better; treat an unmeasured lag (0) as worst. - fn lag_key(p: &ConnectedPeer) -> u32 { - match p.lag_ms() { - 0 => u32::MAX, - ms => ms, - } - } - - let mut candidates = discoverer.get(PROBE_BATCH).await; - // Cap how many candidates we handshake before settling. This bounds startup: - // real peers often ping over 100ms, so without a limit the "wait for fast - // peers" loop would probe the entire batch. We try a few times `max_peers` - // and then take the lowest-latency of whatever connected. - let attempt_max = (max_peers * 3).max(CONNECT_CHUNK * 2); - let mut attempted = 0usize; - let mut kept: Vec = Vec::new(); - let mut fallback: Vec = Vec::new(); // connected, not fast enough - let mut tip = 0u32; - - // Connect a chunk at a time and STOP as soon as we have `max_peers` fast - // peers (or we've tried enough), so sync starts without waiting on the whole - // probe batch. We keep only the peers we'll actually query connected; - // everything else stays a bare address for on-demand recruitment. - while kept.len() < max_peers && !candidates.is_empty() && attempted < attempt_max { - let take = CONNECT_CHUNK.min(candidates.len()); - attempted += take; - let batch: Vec = candidates.drain(..take).collect(); - let results = join_all(batch.into_iter().map(|c| { - c.connect(inbound.clone(), shutdown.clone(), bytes.clone(), required_services) - })) - .await; - for peer in results.into_iter().filter_map(Result::ok) { - tip = tip.max(peer.version().start_height.max(0) as u32); - let lag = peer.lag_ms(); - if kept.len() < max_peers && lag > 0 && lag < FAST_LAG_MS { - kept.push(peer); - } else { - fallback.push(peer); - } - } - } - - // If too few peers were fast, fill up to `max_peers` with the lowest-latency - // fallbacks; close and remember the rest as backups. - fallback.sort_by_key(lag_key); - while kept.len() < max_peers && !fallback.is_empty() { - kept.push(fallback.remove(0)); - } - let mut backups: Vec = candidates; // un-probed addresses - for peer in fallback { - peer.close(); // drop the connection we won't use - backups.push(peer.disconnect()); - } - - let fast = kept.iter().filter(|p| p.lag_ms() > 0 && p.lag_ms() < FAST_LAG_MS).count(); - tracing::info!( - target: "dash_spv::network", - "probe: connected {} peers ({} fast <{}ms), {} backups | tip={}", - kept.len(), - fast, - FAST_LAG_MS, - backups.len(), - tip, - ); - - let new_connected: Vec<(ConnectedPeer, State)> = - kept.into_iter().map(|peer| (peer, State {})).collect(); - // Swap in the probe's selection, but don't strand requests the sync already - // put on the peers the reconnector raced in during startup: drain those in - // the background instead of dropping them here (see `retire_drained`). - let displaced = std::mem::replace(&mut *connected.lock().await, new_connected); - for (peer, _) in displaced { - retire_drained(peer, shutdown.clone()); - } - *others.lock().await = backups; - - tip -} - fn spawn_pump( mut inbound: UnboundedReceiver, subscribers: Subscribers, diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 6c9d229d4..8db93a17d 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -142,6 +142,10 @@ pub struct ConnectedPeer { pub struct DisconnectedPeer { network: Network, addr: SocketAddr, + /// Handshake ping measured the last time we were connected to this peer, if any. + /// Lets the supervisor rank backups by measured quality instead of treating every + /// disconnected address as an unknown. `None` for an address we have never probed. + lag_ms: Option, } impl ConnectedPeer { @@ -164,6 +168,9 @@ impl ConnectedPeer { DisconnectedPeer { network: self.network, addr: self.addr, + // Carry the measured handshake ping (0 means unmeasured) so the supervisor + // can rank this address against others without re-probing it. + lag_ms: (self.lag_ms() > 0).then(|| self.lag_ms()), } } @@ -280,6 +287,7 @@ impl DisconnectedPeer { DisconnectedPeer { network, addr, + lag_ms: None, } } @@ -287,6 +295,12 @@ impl DisconnectedPeer { self.addr } + /// Handshake ping measured on a prior connection, if this address has been probed + /// before. `None` sorts as worst (unknown) when ranking backups. + pub(crate) fn lag_ms(&self) -> Option { + self.lag_ms + } + #[allow(clippy::too_many_arguments)] pub async fn connect( self, From 5581f1e92b44fb678b181624f3a861b7a9f0cb52 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Sat, 18 Jul 2026 11:09:55 +0000 Subject: [PATCH 24/25] testnet scenario using 3 peers --- dash-spv-bench/scenarios/real-testnet.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dash-spv-bench/scenarios/real-testnet.yml b/dash-spv-bench/scenarios/real-testnet.yml index c942f0217..3cee92921 100644 --- a/dash-spv-bench/scenarios/real-testnet.yml +++ b/dash-spv-bench/scenarios/real-testnet.yml @@ -1,4 +1,4 @@ -description: "Testnet sync from a 500k checkpoint via DNS peer discovery." +description: "Testnet sync from a genesis via DNS peer discovery." cpus: "0-11" -max_peers: 16 +max_peers: 3 mode: testnet From 39b7db23fa61f6cc167c550eede42daaabd18e98 Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Sat, 18 Jul 2026 14:09:55 +0000 Subject: [PATCH 25/25] improved peer work distribution with A/B testing made by Claude --- dash-spv-bench/scenarios/slow-many-peers.yml | 11 +++++++++ dash-spv/src/network/manager.rs | 25 ++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 dash-spv-bench/scenarios/slow-many-peers.yml diff --git a/dash-spv-bench/scenarios/slow-many-peers.yml b/dash-spv-bench/scenarios/slow-many-peers.yml new file mode 100644 index 000000000..c0808de3a --- /dev/null +++ b/dash-spv-bench/scenarios/slow-many-peers.yml @@ -0,0 +1,11 @@ +description: "Many slow peers: 8 peers @ 300ms/30ms jitter, 8 Mbit each, full 1M sync. Stresses per-peer cap growth — saturating a high-latency rate-capped link needs several in-flight requests per peer, so caps must climb well above the floor." +mode: local +cpus: "0-5" +max_peers: 8 +repeats: 2 +blocks: 1000000 +peers: + - count: 8 + latency_ms: 300 + jitter_ms: 30 + rate_kbit: 8000 diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index 18a7f831f..704e50fe8 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -689,6 +689,10 @@ struct PeerCapState { /// Uncongested service-time baseline in seconds (the peer's min `W`), 0 until /// first measured. Little's Law targets `L = λ · min_W`. min_w: f64, + /// Windows since `min_w` last took a new low (BBR-style min-filter age). Lets a + /// stale baseline expire and re-track the current cost, instead of one cheap + /// early sample pinning it forever. + min_w_age: u32, /// Smoothed cap, so it doesn't jitter window to window. cap_ema: f64, } @@ -744,6 +748,7 @@ fn spawn_bandwidth_controller( const CAP_GROW: f64 = 1.0; // additive increase per window while lag is flat const CAP_BACKOFF: f64 = 0.8; // multiplicative decrease when lag inflates const W_INFLATE: f64 = 1.5; // W above min_W·this => this peer is backing up + const MIN_W_WINDOW: u32 = 20; // windows before a stale min_W baseline is re-tracked let window_s = WINDOW.as_secs_f64(); tokio::spawn(async move { @@ -851,11 +856,23 @@ fn spawn_bandwidth_controller( } else { let lambda = dc as f64 / window_s; // completions/sec let w = (dt as f64 / dc as f64) / 1e9; // avg service time (s) - st.min_w = if st.min_w == 0.0 { - w + // Windowed min-W baseline (BBR-style min filter): take a new low + // immediately, otherwise let the baseline go stale and re-track + // the current cost after MIN_W_WINDOW windows. Without the reset, + // one low sample from a cheap phase (fast headers) pins the + // baseline forever and every later heavier request (cfilters) + // reads as inflated => the cap decays to the floor and never + // recovers, throttling the very phase we want parallel. + if st.min_w == 0.0 || w < st.min_w { + st.min_w = w; + st.min_w_age = 0; } else { - st.min_w.min(w) - }; + st.min_w_age += 1; + if st.min_w_age >= MIN_W_WINDOW { + st.min_w = w; + st.min_w_age = 0; + } + } // AIMED on this peer's own lag: additive-increase while its // service time sits at the uncongested baseline (headroom), // multiplicative-decrease the moment it inflates (backing up).