From 5011a893ba0175972771d93597662ec4f4308bea Mon Sep 17 00:00:00 2001 From: MoonBoi9001 Date: Fri, 23 Oct 2026 19:27:37 +0200 Subject: [PATCH 1/4] fix(reassess): keep serving members off the denylist sent to IISA IISA now drops any group member on the denylist, cancelling its agreement. An indexer that missed an offer for another deployment went on that list too, so a member still serving this one is now left off it. --- .../handlers/reassess_indexing_request.rs | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs b/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs index 5e6d6068..21a234ab 100644 --- a/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs +++ b/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs @@ -249,7 +249,14 @@ where "unresponsive breaker tripped; skipping this chain's unresponsive exclusion" ); } else { - context.indexer_denylist.extend(unresponsive); + // IISA drops a group member on the denylist, cancelling its agreement here, so a + // member that only missed an offer elsewhere stays off it. + let members: HashSet = context.existing_indexers.iter().copied().collect(); + context.indexer_denylist.extend( + unresponsive + .into_iter() + .filter(|indexer| !members.contains(indexer)), + ); } } @@ -1587,6 +1594,38 @@ mod lifecycle_event_tests { ); } + #[tokio::test] + async fn keeps_a_member_that_missed_an_offer_elsewhere_off_the_denylist() { + // IISA drops a group member on the denylist, which cancels its agreement here, so a + // member that only missed an offer for another deployment must stay off it. + let member = indexer_id(0x40); + let outsider = indexer_id(0x41); + let iisa = MockIisa::default(); + let contexts = Arc::clone(&iisa.contexts); + let ctx = build_ctx( + MockRegistry { + active_agreements: vec![agreement( + member, + IndexingAgreementStatus::AcceptedOnChain, + )], + unresponsive: vec![member, outsider], + ..Default::default() + }, + iisa, + MockQueue::default(), + MockChainClient::default(), + CapturingEventsProducer::new(), + indexer_urls::Snapshot::new(), + ); + + handle(ctx, &test_message(1)).await.expect("handler ok"); + + let contexts = contexts.lock().unwrap(); + assert_eq!(contexts.len(), 1, "IISA asked once: {contexts:?}"); + assert_eq!(contexts[0].existing_indexers, vec![member]); + assert_eq!(contexts[0].indexer_denylist, vec![outsider]); + } + #[tokio::test] async fn retries_a_rejected_selection_after_60_seconds_and_says_it_was_rejected() { // A rejection is usually fixed by a deploy, so the job must stay queued From 3cea0c12c868e631bb569c8330b4b76451778b25 Mon Sep 17 00:00:00 2001 From: MoonBoi9001 Date: Fri, 23 Oct 2026 19:31:46 +0200 Subject: [PATCH 2/4] fix(reassess): send indexers that missed an offer as declined Leaving members off the denylist still mixed 2 meanings in 1 list. Indexers that missed an offer now go in the deployment's declined list, which only stops new offers, so the denylist keeps meaning never use, and the README says it now cancels agreements an indexer holds. --- README.md | 2 +- .../handlers/reassess_indexing_request.rs | 24 ++++++++++--------- dipper-iisa/src/api.rs | 5 ++-- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 96760d62..416dfc2d 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ When an offer fails, dipper temporarily stops offering that indexer new work. Ho | Your price was too high for the deployment | 1 day | | You never replied to the offer | 1 day, on every deployment on that chain | | A transient failure, including offers that never reached the chain | 5 minutes | -| The operator denylist | Until removed | +| The operator denylist | Until removed, and agreements you hold are cancelled | Two safeguards keep this honest. If more than 50% of the DIPs-accepting indexers on a chain stop replying at once, dipper assumes the fault is on its side and suspends the network-wide exclusion until the situation recovers. And a liveness checker (disabled by default) watches accepted agreements for indexers that stop making progress; an abandoned agreement is cancelled on-chain and reassigned. diff --git a/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs b/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs index 21a234ab..e0f3f1ef 100644 --- a/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs +++ b/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs @@ -249,14 +249,14 @@ where "unresponsive breaker tripped; skipping this chain's unresponsive exclusion" ); } else { - // IISA drops a group member on the denylist, cancelling its agreement here, so a - // member that only missed an offer elsewhere stays off it. - let members: HashSet = context.existing_indexers.iter().copied().collect(); - context.indexer_denylist.extend( - unresponsive - .into_iter() - .filter(|indexer| !members.contains(indexer)), - ); + // Sent as declined, which only stops new offers. IISA drops a group member on the + // denylist, which would cancel an agreement it still serves. + let declined = context.declined_indexers.entry(*deployment_id).or_default(); + for indexer in unresponsive { + if !declined.contains(&indexer) { + declined.push(indexer); + } + } } } @@ -1595,9 +1595,9 @@ mod lifecycle_event_tests { } #[tokio::test] - async fn keeps_a_member_that_missed_an_offer_elsewhere_off_the_denylist() { + async fn sends_indexers_that_missed_an_offer_as_declined_not_denylisted() { // IISA drops a group member on the denylist, which cancels its agreement here, so a - // member that only missed an offer for another deployment must stay off it. + // missed offer only stops new offers, for members and outsiders alike. let member = indexer_id(0x40); let outsider = indexer_id(0x41); let iisa = MockIisa::default(); @@ -1623,7 +1623,9 @@ mod lifecycle_event_tests { let contexts = contexts.lock().unwrap(); assert_eq!(contexts.len(), 1, "IISA asked once: {contexts:?}"); assert_eq!(contexts[0].existing_indexers, vec![member]); - assert_eq!(contexts[0].indexer_denylist, vec![outsider]); + assert!(contexts[0].indexer_denylist.is_empty()); + let declined: Vec<_> = contexts[0].declined_indexers.values().flatten().collect(); + assert_eq!(declined, vec![&member, &outsider]); } #[tokio::test] diff --git a/dipper-iisa/src/api.rs b/dipper-iisa/src/api.rs index e48af6c6..bbfa3742 100644 --- a/dipper-iisa/src/api.rs +++ b/dipper-iisa/src/api.rs @@ -27,9 +27,10 @@ pub struct SelectionContext { /// Mapped to `blocklist` in the IISA request. pub indexer_denylist: Vec, - /// For each deployment, indexers that have recently declined agreements. + /// For each deployment, indexers to offer no new agreement for now: those that recently + /// declined, or recently missed an offer on the deployment's chain. /// - /// Used to avoid re-offering agreements to indexers that recently declined. + /// Unlike the denylist, IISA leaves an indexer already in the group in place. /// Key: Deployment ID, Value: List of indexer IDs that declined. pub declined_indexers: HashMap>, From a4c4b8e1a31f4dab93147311684945c7e13594aa Mon Sep 17 00:00:00 2001 From: MoonBoi9001 Date: Tue, 27 Oct 2026 11:41:50 +0100 Subject: [PATCH 3/4] test(selection): check a tripped breaker holds no indexer back When most of a chain's indexers stop replying at once, dipper treats it as its own outage and should not hold them back. Nothing tested that, so breaking it would stop offers on that chain. --- .../handlers/reassess_indexing_request.rs | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs b/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs index dc82f5e5..20b775f8 100644 --- a/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs +++ b/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs @@ -971,6 +971,9 @@ mod lifecycle_event_tests { rejects_with: Option<(u16, &'static str)>, contexts: Arc>>, dips_ceilings: Arc>>, + /// The chain's DIPs-accepting pool, reported as just computed. Empty reports no + /// scores loaded, which leaves the unresponsive breaker inactive. + dips_pool: Vec, } #[async_trait] @@ -997,9 +1000,14 @@ mod lifecycle_event_tests { ceilings: PriceCeilings, ) -> std::result::Result { self.dips_ceilings.lock().unwrap().push(ceilings); + let computed_at = (!self.dips_pool.is_empty()).then(|| { + time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .unwrap() + }); Ok(dipper_iisa::DipsAcceptingSnapshot { - computed_at: None, - indexers: Vec::new(), + computed_at, + indexers: self.dips_pool.clone(), }) } } @@ -1631,6 +1639,37 @@ mod lifecycle_event_tests { assert_eq!(declined, vec![&member, &outsider]); } + #[tokio::test] + async fn holds_back_no_unresponsive_indexer_while_the_breaker_is_tripped() { + // 2 of the chain's 3 DIPs-accepting indexers stopped replying at once, which points at + // dipper rather than at them, so neither is held back from new offers. + let unresponsive = vec![indexer_id(0x50), indexer_id(0x51)]; + let iisa = MockIisa { + dips_pool: vec![indexer_id(0x50), indexer_id(0x51), indexer_id(0x52)], + ..Default::default() + }; + let contexts = Arc::clone(&iisa.contexts); + let ctx = build_ctx( + MockRegistry { + unresponsive, + ..Default::default() + }, + iisa, + MockQueue::default(), + MockChainClient::default(), + CapturingEventsProducer::new(), + indexer_urls::Snapshot::new(), + ); + + handle(ctx, &test_message(1)).await.expect("handler ok"); + + let contexts = contexts.lock().unwrap(); + assert_eq!(contexts.len(), 1, "IISA asked once: {contexts:?}"); + assert!(contexts[0].indexer_denylist.is_empty()); + let declined: Vec<_> = contexts[0].declined_indexers.values().flatten().collect(); + assert!(declined.is_empty(), "{declined:?}"); + } + #[tokio::test] async fn retries_a_rejected_selection_after_60_seconds_and_says_it_was_rejected() { // A rejection is usually fixed by a deploy, so the job must stay queued From f5ced36326311827b34529250bf3927fab1621ad Mon Sep 17 00:00:00 2001 From: MoonBoi9001 Date: Tue, 27 Oct 2026 11:41:55 +0100 Subject: [PATCH 4/4] docs(iisa): list everything the declined indexers sent to IISA include The field's description left out indexers whose agreement dipper is still cancelling, and a leftover line still called them all indexers that declined. --- dipper-iisa/src/api.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dipper-iisa/src/api.rs b/dipper-iisa/src/api.rs index bbfa3742..88db83c5 100644 --- a/dipper-iisa/src/api.rs +++ b/dipper-iisa/src/api.rs @@ -28,10 +28,11 @@ pub struct SelectionContext { pub indexer_denylist: Vec, /// For each deployment, indexers to offer no new agreement for now: those that recently - /// declined, or recently missed an offer on the deployment's chain. + /// declined or missed an offer on the deployment's chain, and those whose agreement on the + /// deployment dipper is still cancelling. /// /// Unlike the denylist, IISA leaves an indexer already in the group in place. - /// Key: Deployment ID, Value: List of indexer IDs that declined. + /// Key: Deployment ID, Value: those indexers' IDs. pub declined_indexers: HashMap>, /// Chain ID of the deployment (e.g., "arbitrum-one").