From 0155f71bd8c8cb0d327a7bd797b48c725bf57e63 Mon Sep 17 00:00:00 2001 From: Dan Anderson Date: Mon, 2 Nov 2026 12:01:28 -0500 Subject: [PATCH] Retire Sequence live Ethereum listings from Radar --- apps/api/src/admin.ts | 8 +- apps/api/src/env.ts | 1 - apps/api/src/indexer/emblem-listings.ts | 122 +----------------- apps/api/src/integrations/sequence.ts | 119 ----------------- apps/api/src/queries/radar.ts | 46 ++----- apps/api/src/read/radar.ts | 2 +- .../src/scheduler/canonical-maintenance.ts | 5 - apps/api/tests/radar-availability.test.ts | 37 ++++++ apps/api/tests/sequence-client.test.ts | 69 ---------- 9 files changed, 52 insertions(+), 357 deletions(-) delete mode 100644 apps/api/src/integrations/sequence.ts create mode 100644 apps/api/tests/radar-availability.test.ts delete mode 100644 apps/api/tests/sequence-client.test.ts diff --git a/apps/api/src/admin.ts b/apps/api/src/admin.ts index 2a51bae9..bab28e7c 100644 --- a/apps/api/src/admin.ts +++ b/apps/api/src/admin.ts @@ -18,7 +18,6 @@ import { crawlScarceSales } from "#api/indexer/scarce-sales"; import { classifyVaults } from "#api/indexer/vault-contents"; import { crawlEmblemMeta } from "#api/indexer/emblem-meta"; import { crawlEmblemTransfers } from "#api/indexer/emblem-transfers"; -import { crawlEmblemListings } from "#api/indexer/emblem-listings"; import { crawlTokenscanCollections } from "#api/indexer/tokenscan-collections"; import { syncCounterTags } from "#api/indexer/counters"; import { buildScamAttribution } from "#api/indexer/emblem-scam"; @@ -194,9 +193,10 @@ admin.post("/admin/crawl-emblem", async (c) => { return c.json(await crawlEmblemStep(c.env)); }); -// Refresh live Emblem listings from the Sequence Marketplace API (rotating contract subset per call). Loop -// this to sweep all ~36 contracts; the cron also advances it ~hourly. -admin.post("/admin/crawl-emblem-listings", async (c) => c.json(await crawlEmblemListings(c.env))); +// Retired explicitly rather than silently running the discontinued provider. +admin.post("/admin/crawl-emblem-listings", (c) => + c.json({ error: "Live Ethereum listings are no longer supported" }, 410), +); // Fold in the tokenscan project directory (~60 collections + sites) as source='tokenscan' collection tags. admin.post("/admin/crawl-tokenscan", async (c) => c.json(await crawlTokenscanCollections(c.env))); diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 8de05a05..8c20aa28 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -17,7 +17,6 @@ export interface Env { ADMIN_TOKEN: string; ALCHEMY_KEY: string; ETHERSCAN_KEY: string; - SEQUENCE_ACCESS_KEY: string; /** Optional: sustains the CMC aggregate calendar via free-tier quotes/latest; job skips when unset. */ CMC_API_KEY?: string; } diff --git a/apps/api/src/indexer/emblem-listings.ts b/apps/api/src/indexer/emblem-listings.ts index b7d87d7c..ad62cae0 100644 --- a/apps/api/src/indexer/emblem-listings.ts +++ b/apps/api/src/indexer/emblem-listings.ts @@ -1,14 +1,4 @@ -/** Live asks for Ethereum NFTs that wrap Counterparty assets, sourced from Sequence Marketplace. */ -import type { Env } from "#api/env"; -import { fetchSequenceListingsPage, SequenceUnregisteredCollection } from "#api/integrations/sequence"; -import { getCoreStateInt, getCoreStateStringArray, setCoreState } from "#api/indexer/core-state"; - -// 40 pages × 100 = a 4,000-listing ceiling per contract. Five pages proved real-world short: the -// curated-fakes contract (0x7e6027…, ~1,700 vaults) carries 500+ live OpenSea listings, and the cap -// throw stalled the whole rotation at its cursor for a day. A page is one subrequest; forty stays -// trivial inside the invocation budget. -const MAX_PAGES_PER_CONTRACT = 40; -const CONTRACTS_PER_RUN = 6; +/** Historical listing projection retained for recovery tooling; live ingestion is retired. */ const MAP_CHUNK = 90; export interface Ask { @@ -25,51 +15,6 @@ export interface ListingRow extends Ask { assetId: number | null; } -/** A contract sweep is accepted only when every provider page is present. */ -async function sweepContract(key: string, contract: string): Promise { - const asks: Ask[] = []; - for (let page = 1; page <= MAX_PAGES_PER_CONTRACT; page++) { - const response = await fetchSequenceListingsPage(key, contract, page); - for (const item of response.collectibles ?? []) { - const order = item.listing ?? item.order; - const tokenId = order?.tokenId ?? item.metadata?.tokenId; - if (!order || !tokenId || order.priceUSD == null) continue; - const expiry = order.validUntil ? Math.floor(new Date(order.validUntil).getTime() / 1000) : 0; - asks.push({ - tokenId: String(tokenId), - orderId: order.orderId ?? null, - marketplace: String(order.marketplace ?? ""), - priceUsd: order.priceUSD, - priceAmount: order.priceAmount ?? null, - currency: (order.priceCurrencyAddress ?? "").toLowerCase(), - expiry: Number.isFinite(expiry) ? expiry : 0, - }); - } - if (!response.page?.more) return asks; - } - throw new Error(`Sequence listing sweep exceeded ${MAX_PAGES_PER_CONTRACT} pages for ${contract}`); -} - -async function resolveAssets(db: D1Database, contract: string, asks: Ask[]): Promise { - const assetByToken = new Map(); - const tokenIds = [...new Set(asks.map((ask) => ask.tokenId))]; - for (let index = 0; index < tokenIds.length; index += MAP_CHUNK) { - const chunk = tokenIds.slice(index, index + MAP_CHUNK); - const rows = await db - .prepare( - `SELECT vault.token_id,vault.contents_asset_id - FROM emblem_vaults vault - JOIN address_dictionary contract ON contract.address_id=vault.contract_id - WHERE contract.address=? AND vault.token_id IN (${chunk.map(() => "?").join(",")})`, - ) - .bind(contract, ...chunk) - .all<{ token_id: string; contents_asset_id: number | null }>(); - for (const row of rows.results) - if (row.contents_asset_id != null) assetByToken.set(row.token_id, row.contents_asset_id); - } - return asks.map((ask) => ({ ...ask, assetId: assetByToken.get(ask.tokenId) ?? null })); -} - /** Reconcile one fully observed contract inside an unpublished generation. */ export async function upsertEmblemListingContract( db: D1Database, @@ -129,68 +74,3 @@ export async function upsertEmblemListingContract( .bind(generation, contract, observedAt) .run(); } - -/** Advance a complete generation in bounded contract groups; publish only after every contract succeeds. */ -export async function crawlEmblemListings(env: Env): Promise> { - if (!env.SEQUENCE_ACCESS_KEY) return { skipped: "no SEQUENCE_ACCESS_KEY" }; - const contracts = await getCoreStateStringArray(env.CORE_DB, "emblem_contracts"); - if (contracts.length === 0) return { skipped: "no contracts" }; - - const published = await getCoreStateInt(env.CORE_DB, "emblem_listings_generation"); - const generation = published + 1; - let cursor = await getCoreStateInt(env.CORE_DB, "emblem_listings_contract_cursor"); - if (cursor < 0 || cursor >= contracts.length) cursor = 0; - let live = 0; - let upserts = 0; - let unregistered = 0; - const processed: string[] = []; - - for (let count = 0; count < CONTRACTS_PER_RUN && cursor < contracts.length; count++) { - const contract = contracts[cursor].toLowerCase(); - let asks: Ask[]; - try { - asks = await sweepContract(env.SEQUENCE_ACCESS_KEY, contract); - } catch (error) { - // A contract the Sequence project doesn't include is a complete, honestly-empty observation — - // most vault contracts are unregistered, and aborting on the first one wedged the venue at - // generation zero while the registered contracts carried real (OpenSea-sourced) listings. - // Anything else is a provider failure: abort so an incomplete generation never publishes. - if (error instanceof SequenceUnregisteredCollection) { - asks = []; - unregistered++; - } else { - return { generation, processed: processed.length, failed: contract, error: String(error).slice(0, 120) }; - } - } - const observedAt = Date.now(); - const rows = await resolveAssets(env.CORE_DB, contract, asks); - await upsertEmblemListingContract(env.CORE_DB, generation, contract, rows, observedAt); - cursor++; - await setCoreState(env.CORE_DB, "emblem_listings_contract_cursor", cursor); - live += rows.length; - upserts += rows.length; - processed.push(contract); - } - - if (cursor >= contracts.length) { - await env.CORE_DB.batch([ - env.CORE_DB.prepare( - `INSERT INTO core_state(key,value) VALUES('emblem_listings_generation',?) - ON CONFLICT(key) DO UPDATE SET value=excluded.value`, - ).bind(String(generation)), - env.CORE_DB.prepare( - `INSERT INTO core_state(key,value) VALUES('emblem_listings_contract_cursor','0') - ON CONFLICT(key) DO UPDATE SET value=excluded.value`, - ), - env.CORE_DB.prepare(`DELETE FROM emblem_listings WHERE generation= contracts.length, - processed: processed.length, - unregistered, - live, - upserts, - }; -} diff --git a/apps/api/src/integrations/sequence.ts b/apps/api/src/integrations/sequence.ts deleted file mode 100644 index 244b7ab4..00000000 --- a/apps/api/src/integrations/sequence.ts +++ /dev/null @@ -1,119 +0,0 @@ -const LISTINGS_URL = "https://marketplace-api.sequence.app/mainnet/rpc/Marketplace/ListCollectiblesWithLowestListing"; -const REQUEST_TIMEOUT_MS = 20_000; -const PAGE_SIZE = 100; - -export interface SequenceOrder { - orderId?: string; - marketplace?: string | number; - tokenId?: string | null; - priceUSD?: number; - priceAmount?: string; - priceCurrencyAddress?: string; - validUntil?: string; -} - -export interface SequenceCollectibleOrder { - metadata?: { tokenId?: string }; - order?: SequenceOrder | null; - listing?: SequenceOrder | null; -} - -export interface SequenceListingsPage { - collectibles?: SequenceCollectibleOrder[]; - page?: { more?: boolean }; - error?: string; - msg?: string; -} - -function isObject(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function optionalType(value: unknown, type: "string" | "number" | "boolean"): boolean { - return value === undefined || typeof value === type; -} - -function isOrder(value: unknown): value is SequenceOrder { - if (!isObject(value)) return false; - return ( - optionalType(value.orderId, "string") && - (value.marketplace === undefined || - typeof value.marketplace === "string" || - typeof value.marketplace === "number") && - (value.tokenId === undefined || value.tokenId === null || typeof value.tokenId === "string") && - optionalType(value.priceUSD, "number") && - (value.priceUSD === undefined || Number.isFinite(value.priceUSD)) && - optionalType(value.priceAmount, "string") && - optionalType(value.priceCurrencyAddress, "string") && - optionalType(value.validUntil, "string") - ); -} - -export function parseSequenceListingsPage(value: unknown): SequenceListingsPage { - if (!isObject(value)) throw new Error("Sequence listings response must be an object"); - if (!optionalType(value.error, "string") || !optionalType(value.msg, "string")) { - throw new Error("Sequence listings response has invalid error fields"); - } - if (value.page !== undefined && (!isObject(value.page) || !optionalType(value.page.more, "boolean"))) { - throw new Error("Sequence listings response has an invalid page"); - } - if (value.collectibles !== undefined) { - if (!Array.isArray(value.collectibles)) throw new Error("Sequence collectibles must be an array"); - for (const [index, collectible] of value.collectibles.entries()) { - if (!isObject(collectible)) throw new Error(`Sequence collectible ${index} must be an object`); - if ( - collectible.metadata !== undefined && - (!isObject(collectible.metadata) || !optionalType(collectible.metadata.tokenId, "string")) - ) { - throw new Error(`Sequence collectible ${index} has invalid metadata`); - } - for (const field of ["order", "listing"] as const) { - const order = collectible[field]; - if (order !== undefined && order !== null && !isOrder(order)) { - throw new Error(`Sequence collectible ${index} has an invalid ${field}`); - } - } - } - } - return value as SequenceListingsPage; -} - -export function requireSequenceListingsPage(value: unknown): SequenceListingsPage { - const page = parseSequenceListingsPage(value); - if (page.error) throw new Error(`Sequence listings error: ${page.error.slice(0, 120)}`); - return page; -} - -/** The contract is not registered in the Sequence project — a permanent, per-contract condition - * (their marketplace API only serves collections added in the builder console), distinct from a - * provider failure. Callers treat it as an honestly empty observation. */ -export class SequenceUnregisteredCollection extends Error { - constructor(contract: string) { - super(`Sequence project does not include ${contract}`); - } -} - -export async function fetchSequenceListingsPage( - accessKey: string, - contract: string, - page: number, -): Promise { - const response = await fetch(LISTINGS_URL, { - method: "POST", - headers: { "Content-Type": "application/json", "X-Access-Key": accessKey }, - body: JSON.stringify({ - contractAddress: contract, - filter: { includeEmpty: false }, - page: { page, pageSize: PAGE_SIZE }, - }), - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); - if (!response.ok) { - const body = await response.text().catch(() => ""); - if (response.status === 400 && /collection not found/i.test(body)) { - throw new SequenceUnregisteredCollection(contract); - } - throw new Error(`Sequence listings request failed: ${response.status}`); - } - return requireSequenceListingsPage(await response.json()); -} diff --git a/apps/api/src/queries/radar.ts b/apps/api/src/queries/radar.ts index c30d21b3..e3335696 100644 --- a/apps/api/src/queries/radar.ts +++ b/apps/api/src/queries/radar.ts @@ -38,11 +38,8 @@ export function radarEstablished(db: D1Database, limit = 40): Promise { return q( db, @@ -68,29 +65,12 @@ export function radarAvailable(db: D1Database, limit = 40): Promise 0) GROUP BY dispenser.asset_id ), - emb_ranked AS ( -- cheapest current-generation listing, keeping venue and URL from the same order - SELECT asset_id,price_usd ask_usd,marketplace,url, - ROW_NUMBER() OVER (PARTITION BY asset_id ORDER BY price_usd,order_id) rank - FROM emblem_listings - WHERE generation=COALESCE((SELECT CAST(value AS INTEGER) FROM core_state - WHERE key='emblem_listings_generation'),0) - AND asset_id IS NOT NULL AND price_usd IS NOT NULL - AND (expiry=0 OR expiry>=unixepoch()) - ), - emb AS ( - SELECT asset_id,ask_usd,marketplace,url FROM emb_ranked WHERE rank=1 - ), - offers AS MATERIALIZED ( - SELECT asset_id FROM disp - UNION - SELECT asset_id FROM emb - ), conv AS ( SELECT signal.asset_id,dictionary.asset, (${CONVICTION}) conviction, COALESCE(signal.max_realized_usd,0) market_usd, signal.holders,signal.supply,signal.avg_holder_dex,signal.pct_creator_holders - FROM offers - JOIN asset_signals signal ON signal.asset_id=offers.asset_id + FROM disp + JOIN asset_signals signal ON signal.asset_id=disp.asset_id JOIN asset_dictionary dictionary ON dictionary.asset_id=signal.asset_id WHERE ${ELIGIBLE} AND (${CONVICTION}) >= ${CONVICTION_PCT.p90} ) @@ -98,21 +78,13 @@ export function radarAvailable(db: D1Database, limit = 40): Promise ); radar.get("/v2/radar", (c) => - cached(c, "radar:established", { ttl: 600, edge: 120 }, async (): Promise> => { + cached(c, "radar:established:dispenser-only", { ttl: 600, edge: 120 }, async (): Promise> => { const [establishedRows, availableRows] = await Promise.all([ radarEstablished(c.env.CORE_DB), radarAvailable(c.env.CORE_DB), diff --git a/apps/api/src/scheduler/canonical-maintenance.ts b/apps/api/src/scheduler/canonical-maintenance.ts index 130564f7..85fc4c13 100644 --- a/apps/api/src/scheduler/canonical-maintenance.ts +++ b/apps/api/src/scheduler/canonical-maintenance.ts @@ -7,7 +7,6 @@ import { crawlCollections } from "#api/indexer/collections"; import { runCoreAssetSignalsStep } from "#api/indexer/core-asset-signals"; import { buildCuratedCollections } from "#api/indexer/curated-collections"; import { reconcileRecentDailyTransactions } from "#api/indexer/daily-metrics"; -import { crawlEmblemListings } from "#api/indexer/emblem-listings"; import { crawlEmblemMeta } from "#api/indexer/emblem-meta"; import { buildScamAttribution } from "#api/indexer/emblem-scam"; import { crawlEmblemTransfers } from "#api/indexer/emblem-transfers"; @@ -66,9 +65,6 @@ async function blockGatedProviderJob( return result as Record; } -const maybeCrawlEmblemListings = (env: Env) => - blockGatedProviderJob(env, "emblem_listings_synced_blk", 6, () => crawlEmblemListings(env)); - const maybeRebuildTags = (env: Env) => runCoreBlockGated(env.CORE_DB, "tags_rebuilt_blk", 144, async () => { await buildTags(env); @@ -118,7 +114,6 @@ export async function runCanonicalMaintenance(env: Env): Promise { await runScheduledJob("crawlTokenscan", () => maybeCrawlTokenscan(env)); await runScheduledJob("crawlEmblemTransfers", () => crawlEmblemTransfers(env)); await runScheduledJob("backfillEthereumBlockTimes", () => backfillEthereumBlockTimes(env)); - await runScheduledJob("crawlEmblemListings", () => maybeCrawlEmblemListings(env)); await runScheduledJob("crawlScarceSales", () => crawlScarceSales(env)); await runScheduledJob("classifyVaults", () => classifyVaults(env)); await runScheduledJob("crawlEmblemMeta", () => crawlEmblemMeta(env)); diff --git a/apps/api/tests/radar-availability.test.ts b/apps/api/tests/radar-availability.test.ts new file mode 100644 index 00000000..05d300fc --- /dev/null +++ b/apps/api/tests/radar-availability.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { DatabaseSync } from "node:sqlite"; +import { test } from "node:test"; +import { radarAvailable } from "#api/queries/radar"; + +test("availability ignores retained Ethereum asks and keeps stocked dispenser unit prices", async () => { + const db = new DatabaseSync(":memory:"); + db.exec(` + CREATE TABLE prices(currency TEXT,day TEXT,usd REAL); + INSERT INTO prices VALUES('BTC','2026-11-02',100000); + CREATE TABLE dispensers(asset_id INTEGER,oracle_address_id INTEGER,satoshirate_normalized REAL,satoshirate INTEGER,give_quantity_normalized REAL,status INTEGER,give_remaining_normalized REAL); + CREATE TABLE broadcasts(source_id INTEGER,value REAL,block_index INTEGER,tx_index INTEGER,status TEXT); + CREATE TABLE assets(asset_id INTEGER,type TEXT,asset_longname TEXT); + CREATE TABLE asset_dictionary(asset_id INTEGER,asset TEXT); + CREATE TABLE asset_signals(asset_id INTEGER,low_quality INTEGER,holders INTEGER,supply REAL,burned_pct REAL,avg_holder_dex REAL,pct_creator_holders REAL,holder_breadth REAL,max_realized_usd REAL,top1_pct REAL); + CREATE TABLE core_state(key TEXT,value TEXT); + INSERT INTO core_state VALUES('emblem_listings_generation','1'); + CREATE TABLE emblem_listings(asset_id INTEGER,price_usd REAL,marketplace TEXT,url TEXT,order_id TEXT,generation INTEGER,expiry INTEGER); + INSERT INTO emblem_listings VALUES(1,1,'opensea','https://opensea.io/old','one',1,0),(2,1,'opensea','https://opensea.io/old','two',1,0); + INSERT INTO assets VALUES(1,'named',NULL),(2,'named',NULL),(3,'named',NULL); + INSERT INTO asset_dictionary VALUES(1,'ETHONLY'),(2,'BOTH'),(3,'EMPTY'); + INSERT INTO asset_signals SELECT asset_id,0,100,100,0,100000,100,10000,1000,0 FROM assets; + INSERT INTO dispensers VALUES(2,NULL,0.002,200000,2,0,4),(3,NULL,0.001,100000,1,0,0); + `); + const d1 = { + prepare: (sql: string) => ({ bind: () => ({ all: async () => ({ results: db.prepare(sql).all() }) }) }), + } as unknown as D1Database; + const rows = await radarAvailable(d1); + assert.equal(rows.length, 1); + assert.equal(rows[0].asset, "BOTH"); + assert.equal(rows[0].venue, "dispenser"); + assert.equal(rows[0].ask_btc, 0.001); + assert.equal(rows[0].ask_usd, 100); + assert.equal(rows[0].listing_url, null); + assert.equal(rows[0].marketplace, null); + db.close(); +}); diff --git a/apps/api/tests/sequence-client.test.ts b/apps/api/tests/sequence-client.test.ts deleted file mode 100644 index aa8d2176..00000000 --- a/apps/api/tests/sequence-client.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { - fetchSequenceListingsPage, - parseSequenceListingsPage, - requireSequenceListingsPage, - SequenceUnregisteredCollection, -} from "#api/integrations/sequence"; - -test("Sequence listings parsing accepts the consumed provider shape", () => { - const response = { - collectibles: [ - { - metadata: { tokenId: "42" }, - listing: { orderId: "order", priceUSD: 12.5, marketplace: "opensea" }, - }, - ], - page: { more: true }, - }; - assert.deepEqual(parseSequenceListingsPage(response), response); -}); - -test("Sequence listings parsing accepts provider error envelopes", () => { - assert.deepEqual(parseSequenceListingsPage({ error: "not found", msg: "missing" }), { - error: "not found", - msg: "missing", - }); - assert.throws(() => requireSequenceListingsPage({ error: "upstream unavailable" }), /upstream unavailable/); -}); - -test("Sequence listings parsing rejects provider drift", () => { - assert.throws(() => parseSequenceListingsPage([]), /must be an object/); - assert.throws(() => parseSequenceListingsPage({ collectibles: {} }), /must be an array/); - assert.throws(() => parseSequenceListingsPage({ page: { more: "yes" } }), /invalid page/); - assert.throws( - () => parseSequenceListingsPage({ collectibles: [{ listing: { priceUSD: "12.5" } }] }), - /invalid listing/, - ); -}); - -test("an unregistered collection raises its typed condition; other failures stay generic", async () => { - const originalFetch = globalThis.fetch; - const respond = (status: number, body: string) => { - globalThis.fetch = async () => new Response(body, { status }); - }; - try { - respond( - 400, - JSON.stringify({ error: "NotFound", code: 2000, cause: "collection not found contractAddress=(0xabc)" }), - ); - const rejection = async (): Promise => { - let error: unknown = null; - try { - await fetchSequenceListingsPage("key", "0xabc", 1); - } catch (caught) { - error = caught; - } - assert.ok(error !== null, "expected the fetch to reject"); - return error; - }; - assert.ok((await rejection()) instanceof SequenceUnregisteredCollection); - respond(400, JSON.stringify({ error: "BadRequest", cause: "page out of range" })); - assert.match(String(await rejection()), /request failed: 400/); - respond(503, "upstream down"); - assert.match(String(await rejection()), /request failed: 503/); - } finally { - globalThis.fetch = originalFetch; - } -});