diff --git a/src/main.tsx b/src/main.tsx index 1dbaee79..1826ac53 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -124,6 +124,10 @@ const SORTS_KEY = 'spdr-tab-sorts'; const SITE_STATE_KEY = 'spdr-site-state'; const HOLDINGS_CONCURRENCY = 6; // bounded whole-catalog aggregation workers const WATCHLIST_CHUNK = 250; // rows per rendered Watchlist DOM chunk +const SHEET_CHUNK = 300; // rows per rendered Holdings / History DOM chunk (loaded pages beyond it mount on scroll) +let sheetChunkSig = ''; +let sheetRenderedCount = 0; +let sheetShownTotal = 0; // filtered rows of the shown sheet at the last render (how far the DOM window can grow) const DEFAULT_SELECTED_TICKERS: string[] = []; // start clean: no pre-selected funds const DETAIL_TABS: Array<{ key: string; label: string }> = [ @@ -277,6 +281,9 @@ const state: AppState = { const metaInFlight: Map> = new Map(); const holdingsChains: Map> = new Map(); const holdingsComplete = new Set(); +const holdingsInFlight = new Set(); +const holdingsFailed = new Set(); // fetch failed: not complete, retried after the fund is selected again +const metaFailures = new Set(); let watchlistChunkSig = ''; let watchlistRenderedCount = 0; let watchlistRefreshTimer: any = null; @@ -1102,12 +1109,21 @@ function filteredCatalogFunds(): FundRow[] { } /** Sheet rows (holdings, history, distributions) as objects with the cells in col0..colN, as shown (search, sort, column filters). */ +/** Row objects of a sheet are built once per source row and reused while pages are appended in place (see fetchNextSheetPage). */ +const sheetRowObjects: WeakMap = new WeakMap(); + function sheetView(scope: string, headers: string[], sourceRows: string[][], numericHeaders: string[]): any[] { - return tableView(scope, sheetFilterColumns(headers, numericHeaders), sourceRows, '', () => sourceRows.map((row, sourceIndex) => { - const cells: Record = { values: row, searchIndex: row.join(' ').toLowerCase(), rank: sourceIndex }; - headers.forEach((header, index) => { cells[`col${index}`] = row[index] ?? ''; }); - return cells; - })); + return tableView(scope, sheetFilterColumns(headers, numericHeaders), sourceRows, '', () => { + let built = sheetRowObjects.get(sourceRows); + if (!built || built.length > sourceRows.length) { built = []; sheetRowObjects.set(sourceRows, built); } + for (let sourceIndex = built.length; sourceIndex < sourceRows.length; sourceIndex++) { + const row = sourceRows[sourceIndex]; + const cells: Record = { values: row, searchIndex: row.join(' ').toLowerCase(), rank: sourceIndex }; + for (let index = 0; index < headers.length; index++) cells[`col${index}`] = row[index] ?? ''; + built.push(cells); + } + return built.slice(); + }); } // ========================================================================= @@ -1774,10 +1790,18 @@ function setStatus(message: string, tone: 'info' | 'success' | 'error'): void { // 4. Static API loading & paginated sheets (api/spdr/**, iShares-style) // ========================================================================= -async function fetchJson(url: string): Promise { - const response = await fetch(url, { headers: { Accept: 'application/json' }, cache: 'no-cache' }); - if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); - return response.json(); +const FETCH_TIMEOUT_MS = 30000; // index.json, meta.json and sheet pages (headers and body) + +async function fetchJson(url: string, timeoutMs = FETCH_TIMEOUT_MS): Promise { + const controller = typeof AbortController === 'function' ? new AbortController() : null; + const timer = controller && timeoutMs > 0 ? setTimeout(() => controller.abort(), timeoutMs) : null; + try { + const response = await fetch(url, { headers: { Accept: 'application/json' }, cache: 'no-cache', signal: controller ? controller.signal : undefined }); + if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); + return await response.json(); + } finally { + if (timer !== null) clearTimeout(timer); + } } /** Flattens index.json month-end metrics onto the row so sorting works. */ @@ -1864,9 +1888,11 @@ async function loadFundMeta(ticker: string): Promise { const request = (async () => { try { const meta = await fetchJson(`./api/spdr/funds/${encodeURIComponent(ticker)}/meta.json`); + metaFailures.delete(ticker); fundMetaCache.set(ticker, meta); return meta; } catch (error) { + metaFailures.add(ticker); console.warn(`Failed to load meta.json for ${ticker}:`, error); return null; } @@ -1918,7 +1944,7 @@ async function fetchNextSheetPage(ticker: string, entry: SheetEntry): Promise= entry.manifest.pages.length) return; const page = await fetchPage(ticker, entry.manifest.pages[entry.nextPage]); if (!entry.headers.length && page.headers.length) entry.headers = page.headers; - entry.rows = entry.rows.concat(page.rows); + for (let i = 0; i < page.rows.length; i++) entry.rows.push(page.rows[i]); // in place: no O(n) copy per page, same array identity for the memoized views entry.nextPage += 1; } @@ -1950,7 +1976,7 @@ async function loadNextSheetPage(sheet: 'holdings' | 'history'): Promise { /** True while any selected ETF's holdings have not finished loading. */ function isHoldingsLoading(): boolean { for (const ticker of state.selected) { - if (!holdingsComplete.has(ticker)) return true; + if (!holdingsComplete.has(ticker) && !holdingsFailed.has(ticker)) return true; } return false; } @@ -1962,11 +1988,15 @@ function isHoldingsLoading(): boolean { * the meantime is skipped; in-flight loads finish into the cache and are * ignored by the aggregation, which only reads the current selection. */ -async function loadAllHoldingsForTicker(ticker: string): Promise { - await withTickerChain(ticker, async () => { - if (!state.selected.has(ticker)) return; // deselected while queued: skip +async function loadAllHoldingsForTicker(ticker: string): Promise { + return withTickerChain(ticker, async () => { + if (!state.selected.has(ticker)) return false; // deselected while queued: resume on the next selection const meta = await loadFundMeta(ticker); - if (!meta || !meta.holdings || !Array.isArray(meta.holdings.pages) || !meta.holdings.pages.length) return; + if (!meta) { + if (metaFailures.has(ticker)) throw new Error('meta.json unavailable'); + return true; // catalog-only fund: no holdings published + } + if (!meta.holdings || !Array.isArray(meta.holdings.pages) || !meta.holdings.pages.length) return true; const key = `${ticker}:holdings`; let entry = sheetState.get(key); if (!entry) { @@ -1974,9 +2004,10 @@ async function loadAllHoldingsForTicker(ticker: string): Promise { sheetState.set(key, entry); } while (entry.nextPage < entry.manifest.pages.length) { - if (!state.selected.has(ticker)) return; // deselected mid-load: skip the rest + if (!state.selected.has(ticker)) return false; // deselected mid-load: the partial entry is kept and resumed later await fetchNextSheetPage(ticker, entry); // chain already held: no re-queue } + return true; }); } @@ -1988,18 +2019,21 @@ async function loadAllHoldingsForTicker(ticker: string): Promise { * independent of the active fund. */ async function ensureHoldingsForSelection(): Promise { - const queue = [...state.selected].filter(ticker => !holdingsComplete.has(ticker)); + holdingsFailed.forEach(ticker => { if (!state.selected.has(ticker)) holdingsFailed.delete(ticker); }); // deselect clears a failure: re-select retries + const queue = [...state.selected].filter(ticker => !holdingsComplete.has(ticker) && !holdingsInFlight.has(ticker) && !holdingsFailed.has(ticker)); if (!queue.length) return; + queue.forEach(ticker => holdingsInFlight.add(ticker)); const workers = Array.from({ length: Math.min(HOLDINGS_CONCURRENCY, queue.length) }, async () => { for (;;) { const ticker = queue.shift(); if (!ticker) break; try { - await loadAllHoldingsForTicker(ticker); + if (await loadAllHoldingsForTicker(ticker)) holdingsComplete.add(ticker); } catch (error) { console.error(`Failed to load ${ticker} holdings:`, error); + holdingsFailed.add(ticker); } finally { - holdingsComplete.add(ticker); + holdingsInFlight.delete(ticker); if (state.selected.size > 0) { // Keep the Watchlist tab label honest (Loading… -> N+ -> exact) // even while the user stays on another tab. @@ -2035,9 +2069,20 @@ function activeSheetTab(): 'holdings' | 'history' | null { function maybeLoadMoreRows(): void { const sheet = activeSheetTab(); if (!sheet) return; + if (growSheetChunk()) return; // loaded rows first, the network second void loadNextSheetPage(sheet); } +/** Mounts the next SHEET_CHUNK of already loaded sheet rows; false when every filtered row is in the DOM. */ +function growSheetChunk(): boolean { + const sheet = activeSheetTab(); + if (!sheet || sheetRenderedCount + SHEET_CHUNK >= sheetShownTotal) return false; + sheetRenderedCount += SHEET_CHUNK; + renderDetailTable(sheet); + renderStaticLoadSentinel(); + return true; +} + function renderStaticLoadSentinel(): void { const sheet = activeSheetTab(); if (!sheet || !state.activeFundTicker) { @@ -2049,7 +2094,7 @@ function renderStaticLoadSentinel(): void { el.staticLoadSentinel.classList.add('hidden'); return; } - const more = entry.nextPage < entry.manifest.pages.length; + const more = entry.nextPage < entry.manifest.pages.length || sheetRenderedCount + SHEET_CHUNK < sheetShownTotal; el.staticLoadSentinel.classList.toggle('hidden', !more); el.staticLoadStatus.textContent = entry.loading ? 'Loading more rows…' : more ? 'Scroll or click to load more rows…' : ''; } @@ -2488,18 +2533,38 @@ function compareValues(a: unknown, b: unknown): number { return as.localeCompare(bs, undefined, { numeric: true }); } +const SORT_COLLATOR = new Intl.Collator(undefined, { numeric: true }); +const HISTORY_DATE_RE = /^\d{1,2}-[A-Za-z]{3}-\d{4}$/; + +type SortKeys = { empty: boolean; num: number | null; text: string; date: number }; +type SortKeyed = SortKeys & { row: any }; + +/** The comparison keys of one cell, computed once per row (not once per comparison). */ +function sortKeyOf(value: unknown): SortKeys { + if (isEmptySortValue(value)) return { empty: true, num: null, text: '', date: NaN }; + const num = numberOrNull(value); + const text = String(value ?? ''); + const date = HISTORY_DATE_RE.test(text) ? Date.parse(text.replace(/-/g, ' ')) : NaN; + return { empty: false, num, text, date }; +} + function sortRows(rows: any[]): any[] { if (state.sortKey === 'rank') return rows; const direction = state.sortDir === 'asc' ? 1 : -1; const key = state.sortKey; - return [...rows].sort((a, b) => { - const av = sortValue(a, key); - const bv = sortValue(b, key); - const ae = isEmptySortValue(av); - const be = isEmptySortValue(bv); - if (ae || be) return ae && be ? 0 : ae ? 1 : -1; - return compareValues(av, bv) * direction; + const keyed: SortKeyed[] = new Array(rows.length); + for (let i = 0; i < rows.length; i++) keyed[i] = { row: rows[i], ...sortKeyOf(sortValue(rows[i], key)) }; + keyed.sort((a, b) => { + if (a.empty || b.empty) return a.empty && b.empty ? 0 : a.empty ? 1 : -1; // unavailable values last in both directions + let order: number; + if (a.num !== null && b.num !== null) order = a.num - b.num; + else if (!Number.isNaN(a.date) && !Number.isNaN(b.date)) order = a.date - b.date; // History dates ("21-Aug-2026") chronologically + else order = SORT_COLLATOR.compare(a.text, b.text); + return order * direction; }); + const out: any[] = new Array(keyed.length); + for (let i = 0; i < keyed.length; i++) out[i] = keyed[i].row; + return out; } function isEmptySortValue(value: unknown): boolean { @@ -2651,21 +2716,6 @@ function renderFundsTable(): void { }).join(''); } - el.tableBody.querySelectorAll('input[data-checkbox]').forEach((checkbox: any) => { - checkbox.addEventListener('change', (event: any) => { - event.stopPropagation(); - const ticker = checkbox.dataset.checkbox || ''; - toggleFund(ticker); - }); - checkbox.addEventListener('click', (event: any) => event.stopPropagation()); - }); - - el.tableBody.querySelectorAll('button[data-blacklist]').forEach((button: any) => { - button.addEventListener('click', (event: any) => { - event.stopPropagation(); - blacklistTickers([button.dataset.blacklist || '']); - }); - }); renderCatalogStatus(rows); } @@ -2854,8 +2904,37 @@ function watchlistChunkSignature(rows: WatchlistRow[]): string { function growWatchlistChunk(): void { const rows = getVisibleWatchlistRows(); if (watchlistRenderedCount + WATCHLIST_CHUNK >= rows.length) return; + const from = watchlistRenderedCount + WATCHLIST_CHUNK; watchlistRenderedCount += WATCHLIST_CHUNK; - renderWatchlistTable(); + const moreRow = el.tableBody.querySelector('#watchlist-more-row'); + if (!moreRow || watchlistChunkSignature(rows) !== watchlistChunkSig) { renderWatchlistTable(); return; } // rows changed: full render + const to = Math.min(rows.length, watchlistRenderedCount + WATCHLIST_CHUNK); + moreRow.insertAdjacentHTML('beforebegin', rows.slice(from, to).map((row, i) => watchlistRowHtml(row, from + i)).join('')); + const moreCell = moreRow.querySelector('td'); + if (to >= rows.length) moreRow.remove(); + else if (moreCell) moreCell.textContent = `Scroll or click to load more rows… (${rows.length - to} remaining)`; +} + +/** One Watchlist row; shared by the full render and the append-only growWatchlistChunk. */ +function watchlistRowHtml(row: WatchlistRow, index: number): string { + return ` + + ${rowNumber(row, index)} + ${escapeHtml(row.symbol)} + ${escapeHtml(row.name || '—')} + +
+ ${row.funds.map((ticker: string) => `${escapeHtml(ticker)}`).join('')} +
+ + ${row.fundCount} + ${row.weightSum.toFixed(3)}% + ${row.maxWeight.toFixed(3)}% + ${row.marketValue ? formatMoney(row.marketValue) : '—'} + ${escapeHtml(row.sector || '—')} + ${escapeHtml(row.identifier || '—')} + + `; } function renderWatchlistTable(): void { @@ -2896,24 +2975,7 @@ function renderWatchlistTable(): void { : 'Holdings data is not available yet. Run bun ./scripts/update-data.ts to refresh the SPDR feed.' }${currentQuery() ? ' No rows match your search.' : ''}`; } else { - let html = visibleRows.map((row, index) => ` - - ${rowNumber(row, index)} - ${escapeHtml(row.symbol)} - ${escapeHtml(row.name || '—')} - -
- ${row.funds.map((ticker: string) => `${escapeHtml(ticker)}`).join('')} -
- - ${row.fundCount} - ${row.weightSum.toFixed(3)}% - ${row.maxWeight.toFixed(3)}% - ${row.marketValue ? formatMoney(row.marketValue) : '—'} - ${escapeHtml(row.sector || '—')} - ${escapeHtml(row.identifier || '—')} - - `).join(''); + let html = visibleRows.map((row, index) => watchlistRowHtml(row, index)).join(''); if (loading) { html += `Loading holdings of the remaining selected ETFs…`; } @@ -2923,14 +2985,6 @@ function renderWatchlistTable(): void { el.tableBody.innerHTML = html; } - el.tableBody.querySelectorAll('a[data-activate-fund]').forEach((link: any) => { - link.addEventListener('click', (event: any) => { - event.stopPropagation(); - activateFund(link.dataset.activateFund || ''); - }); - }); - const moreRow = el.tableBody.querySelector('#watchlist-more-row'); - if (moreRow) moreRow.addEventListener('click', () => growWatchlistChunk()); const queryText = currentQuery() ? ` matching “${currentQuery()}”` : ''; setStatus(`Watchlist built from ${state.selected.size} selected ETF${state.selected.size === 1 ? '' : 's'}: ${rows.length} ticker${rows.length === 1 ? '' : 's'}${queryText}. Deduplicated by ticker, or by identifier for bond rows.${rankHint(rows.length)}`, 'success'); @@ -2993,6 +3047,11 @@ function renderSheetTable(fund: FundRow, sheet: 'holdings' | 'history'): void { const headers = entry.headers; const rows = sheetView(sheet, headers, entry.rows, NUMERIC_SHEET_HEADERS); + const chunkSig = [sheetKey(sheet), state.sortKey, state.sortDir, currentQuery(), columnFilterSig(sheet)].join('|'); + if (chunkSig !== sheetChunkSig) { sheetChunkSig = chunkSig; sheetRenderedCount = 0; } + sheetShownTotal = rows.length; + const shownRows = rows.slice(0, sheetRenderedCount + SHEET_CHUNK); + const cellClasses = headers.map(header => NUMERIC_SHEET_HEADERS.includes(header) ? 'text-right font-mono text-slate-700 dark:text-slate-300' : 'text-slate-700 dark:text-slate-300'); el.tableHead.innerHTML = ` @@ -3005,16 +3064,21 @@ function renderSheetTable(fund: FundRow, sheet: 'holdings' | 'history'): void { if (!rows.length) { el.tableBody.innerHTML = `No rows match your search${entry.loading ? ' (still loading…)' : ''}.`; } else { - el.tableBody.innerHTML = rows.map((row, index) => { + const parts: string[] = new Array(shownRows.length); + for (let index = 0; index < shownRows.length; index++) { + const row = shownRows[index]; const values: string[] = row.values || []; - return ` - - ${rowNumber(row, index)} - ${values.map((cell, columnIndex) => ` - ${escapeHtml(cell === '' ? '—' : cell)} - `).join('')} - - `;}).join(''); + let cells = ''; + for (let columnIndex = 0; columnIndex < values.length; columnIndex++) { + const cell = values[columnIndex]; + cells += `${escapeHtml(cell === '' ? '—' : cell)}`; + } + parts[index] = `${rowNumber(row, index)}${cells}`; + } + if (rows.length > shownRows.length) { + parts.push(`Scroll or click to show more rows… (${(rows.length - shownRows.length).toLocaleString('en-US')} loaded rows not shown yet)`); + } + el.tableBody.innerHTML = parts.join(''); } el.tickerCount.textContent = fund.ticker; @@ -4066,6 +4130,26 @@ function bindEvents(): void { }); el.blacklistClearBtn.addEventListener('click', clearBlacklist); + // One delegated listener per event type instead of per-row listeners re-attached on every render. + el.tableBody.addEventListener('change', (event: any) => { + const target = event.target; + const checkbox = target && target.closest ? target.closest('input[data-checkbox]') : null; + if (!checkbox) return; + event.stopPropagation(); + toggleFund(checkbox.dataset.checkbox || ''); + }); + el.tableBody.addEventListener('click', (event: any) => { + const target = event.target; + if (!target || !target.closest) return; + if (target.closest('input[data-checkbox]')) { event.stopPropagation(); return; } + const blacklist = target.closest('button[data-blacklist]'); + if (blacklist) { event.stopPropagation(); blacklistTickers([blacklist.dataset.blacklist || '']); return; } + const link = target.closest('a[data-activate-fund]'); + if (link) { event.stopPropagation(); activateFund(link.dataset.activateFund || ''); return; } + if (target.closest('#watchlist-more-row')) { growWatchlistChunk(); return; } + if (target.closest('#sheet-more-row')) growSheetChunk(); + }); + // Paginated sheets: append more rows as the sentinel scrolls into view. if (typeof IntersectionObserver === 'function') { const observer = new IntersectionObserver( @@ -4087,10 +4171,12 @@ function bindEvents(): void { } const sheet = activeSheetTab(); if (!sheet || !state.activeFundTicker) return; + const distanceToBottom = el.tableScroll.scrollHeight - el.tableScroll.scrollTop - el.tableScroll.clientHeight; + if (distanceToBottom >= 600) return; + if (growSheetChunk()) return; // mount loaded rows before fetching the next page const entry = sheetState.get(sheetKey(sheet)); if (!entry || entry.loading || entry.nextPage >= entry.manifest.pages.length) return; - const distanceToBottom = el.tableScroll.scrollHeight - el.tableScroll.scrollTop - el.tableScroll.clientHeight; - if (distanceToBottom < 600) void loadNextSheetPage(sheet); + void loadNextSheetPage(sheet); }, { passive: true }); // Keep only the table scrolling: refit on viewport changes and whenever