diff --git a/README.md b/README.md index 25a7a9e4..dcbf4c1b 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Each fund carries a derived `metrics` object that powers the catalog columns sha Expense ratio: `terValue` / `ter` are the NET expense ratio (after waivers; the single published figure when there is no waiver), `terGrossValue` / `terGross` the gross ratio. Both come from the product-data workbook ("Gross Expense Ratio", "Net Expense Ratio"), with the fund finder expense ratio as the gross fallback. The `TER` filter uses the net value. -Sources and scope: SPDR has no Yahoo Finance and no SEC EDGAR source. Everything comes from SSGA's own files, so there is no `SKIP_YAHOO`, `EDGAR_FALLBACK` or `SEC_UA` control. Yields are the official 30-day SEC yield and Fund Dividend Yield from SSGA's product-data workbook. +Sources and scope: SPDR has no Yahoo Finance and no SEC EDGAR source. Everything comes from SSGA's own files, so there is no `SKIP_YAHOO`, `EDGAR_FALLBACK` or `SEC_UA` control. Yields are the official 30-day SEC yield and Fund Dividend Yield from SSGA's product-data workbook. When the product-data workbook or the dividend-distribution feed fails or comes back empty, every published fund is kept exactly as published and reported failed; only new funds are built. Identifier columns of the holdings workbooks (Ticker, Identifier, SEDOL) are kept as published text, so CUSIP `037833100` and A-share ticker `002594` keep their leading zeros. Commodity trusts such as GLD and GLDM hold metal and publish no holdings workbook. That is valid: they get `meta.json`, NAV, returns and history, with holdings `status: "empty"` (`holdingsStatus` in `index.json`). A fund that published holdings before is never emptied by a transient 404. diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index af2e4076..c8067e93 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -273,6 +273,17 @@ describe('parsing', () => { expect(bond.table.rows[0][5]).toBe('77000000'); }); + test('identifier columns keep their published text: leading zeros and E-shaped CUSIPs survive, numbers are still expanded', () => { + const sheet = table(buildXlsx([ + ['Fund Name:', 'SPDR S&P China ETF'], ['Ticker Symbol:', 'GXC'], ['Holdings:', 'As of 23-Oct-2026'], + ['Name', 'Ticker', 'Identifier', 'SEDOL', 'Weight', 'Sector', 'Shares Held', 'Local Currency'], + ['BYD CO LTD A', '002594', '037833100', '0263494', '1.5', '-', '1.80135563E8', 'CNY'], + ['ACME CORP', 'ACME', '123456E10', 'B0LMTQ3', '98.5', '-', '5.84E7', 'USD'], + ]), 'holdings'); + expect(sheet.table.rows.map((r) => r.slice(1, 4))).toEqual([['002594', '037833100', '0263494'], ['ACME', '123456E10', 'B0LMTQ3']]); + expect(sheet.table.rows.map((r) => r[6])).toEqual(['180135563', '58400000']); + }); + test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => { const headers = ['Name', 'Ticker', 'Weight']; const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }]; @@ -459,7 +470,7 @@ const holdingsSheet = (t: string) => [['Fund Name:', `${t} fund`], ['Ticker Symb const productSheet = [['Ticker', 'ISIN', 'CUSIP', 'Gross Expense Ratio', '* Net Expense Ratio', '30 Day SEC Yield', '30 Day SEC Yield (Unsubsidized)', 'Fund Dividend Yield', 'Index Dividend Yield'], ['SPY', 'US1', 'C1', '0.0945%', '0.0800%', '0.95%', '-', '1.00%', '1.10%'], ['XLK', 'US2', 'C2', '0.08%', '-', '-', '-', '0.55%', '0.60%'], ['GLD', 'US3', 'C3', '0.40%', '-', '-', '-', '0.00%', '-']]; -type Mock = { tickers: string[]; nav: Record; hist: number; failNavhist: Set; failDistributions: Set; latency: number; inflight: number; peak: number }; +type Mock = { tickers: string[]; nav: Record; hist: number; failNavhist: Set; failDistributions: Set; failBulk: Set; latency: number; inflight: number; peak: number }; // SPY pays quarterly: the lineup feed lists only the latest row, the per-fund history lists all three. const distRow = (exDate: string, dividend: string) => ({ exDate, recordDate: exDate, payableDate: exDate, dividend, shortTeamCapital: '-', longTeamCapital: '-' }); const spyHistory = [distRow('09/18/2026', '1.888834'), distRow('06/18/2026', '1.903516'), distRow('03/20/2026', '1.738591')]; @@ -473,7 +484,7 @@ function fundRecord(ticker: string, nav: number) { return record; } function installMock(overrides: Partial = {}): Mock { - const mock: Mock = { tickers: ['SPY', 'XLK', 'GLD'], nav: { SPY: 500, XLK: 200, GLD: 300 }, hist: 300, failNavhist: new Set(), failDistributions: new Set(), latency: 0, inflight: 0, peak: 0, ...overrides }; + const mock: Mock = { tickers: ['SPY', 'XLK', 'GLD'], nav: { SPY: 500, XLK: 200, GLD: 300 }, hist: 300, failNavhist: new Set(), failDistributions: new Set(), failBulk: new Set(), latency: 0, inflight: 0, peak: 0, ...overrides }; globalThis.fetch = (async (input: unknown) => { const url = String(input); mock.inflight++; mock.peak = Math.max(mock.peak, mock.inflight); @@ -483,6 +494,7 @@ function installMock(overrides: Partial = {}): Mock { if (url.includes('fundfinder')) return Response.json({ data: { funds: { etfs: { categories: [], datas: mock.tickers.map((t) => fundRecord(t, mock.nav[t])) } } } }); const distTicker = /dividend-distribution.*[?&]ticker=([A-Z]+)/.exec(url)?.[1]; if (distTicker) return mock.failDistributions.has(distTicker) ? new Response('err', { status: 500 }) : Response.json(distTicker === 'SPY' ? spyHistory : []); + for (const bulk of ['dividend-distribution', 'product-data']) if (url.includes(bulk) && !url.includes('ticker=') && mock.failBulk.has(bulk)) return new Response('err', { status: 500 }); if (url.includes('dividend-distribution')) return Response.json(distributionFeed); if (url.includes('product-data')) return xlsx(productSheet); const m = /(holdings-daily|navhist|pdhist)-us-en-([a-z]+)\.xlsx/.exec(url); @@ -746,6 +758,23 @@ describe('pipeline 3', () => { expect(order(null)).toEqual(['A', 'C', 'E']); }); + test('a failed or empty bulk feed keeps every published fund as published (failed); a new fund is still built', async () => { + for (const bulk of ['product-data', 'dividend-distribution']) { + const dir = tempRoot(); + installMock(); + await runMain(); + const metas = ['SPY', 'XLK', 'GLD'].map((t) => readFileSync(join(dir, `funds/${t}/meta.json`), 'utf8')); + const spyRow = row(dir, 'SPY'); + const lines: string[] = []; + installMock({ tickers: ['SPY', 'XLK', 'GLD', 'XLE'], nav: { SPY: 777, XLK: 200, GLD: 300, XLE: 90 }, failBulk: new Set([bulk]) }); + await runMain({}, lines); + expect(['SPY', 'XLK', 'GLD'].map((t) => readFileSync(join(dir, `funds/${t}/meta.json`), 'utf8'))).toEqual(metas); + expect(row(dir, 'SPY')).toEqual(spyRow); + expect(lines.join('\n')).toMatch(/SPY +failed .*reason=.*failed, kept as published/); + expect(row(dir, 'XLE').dataFile).toBe('./funds/XLE/meta.json'); + } + }, 20_000); + test('soft deadline: no new funds are started, the index is still written', async () => { const dir = tempRoot(); setSoftDeadlineMs(-1); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index b3aa0164..6fcf06fe 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -747,6 +747,9 @@ export function loadSharedStrings(bytes: Uint8Array): string[] { export type SheetTable = { headers: string[]; rows: string[][] }; +/** Holdings columns that carry identifiers, kept exactly as published (never number-normalized). */ +const IDENTIFIER_HEADERS = /^(ticker|identifier|sedol|cusip|isin)$/i; + /** * Converts raw worksheet rows into a {meta, table} pair. The SPDR workbooks * start with 2-3 metadata rows (Fund Name, Ticker Symbol, holdings as-of) @@ -782,6 +785,8 @@ export function sheetToTable( let width = headerRow.length; while (width > 0 && cleanText(headerRow[width - 1]) === '') width -= 1; const headers = headerRow.slice(0, width).map((cell) => cleanText(cell)); + // Identifiers are text: number normalization would strip leading zeros (CUSIP 037833100, A-share ticker 002594). + const textColumns = new Set(headers.flatMap((header, i) => (IDENTIFIER_HEADERS.test(header) ? [i] : []))); const rows: string[][] = []; for (let i = headerIndex + 1; i < rawRows.length; i += 1) { const row = rawRows[i]; @@ -789,7 +794,7 @@ export function sheetToTable( // Every holding/history row has at least two populated cells; this drops the // trailing legal disclaimer row (one long cell) and blank separator rows. if (!row || cells.filter((cell) => cell !== '').length < 2) continue; - rows.push(cells.map((cell) => normalizeNumberText(cell))); + rows.push(cells.map((cell, i) => (textColumns.has(i) ? cell : normalizeNumberText(cell)))); } return { meta, table: { headers, rows } }; } @@ -1680,7 +1685,7 @@ export async function main(env: Record = process.env try { return normalizeDistributions(await fetchJson(DISTRIBUTIONS_URL, '[distr ] dividend-distribution')); } catch (error) { - console.warn(`[ ${'distr'.padEnd(9)}] dividend feed failed (${(error as Error).message}); continuing without it`); + console.warn(`[ ${'distr'.padEnd(9)}] dividend feed failed (${(error as Error).message}); published funds are kept`); return new Map(); } })(); @@ -1699,12 +1704,21 @@ export async function main(env: Record = process.env } return parseProductDataSheet(bytes); } catch (error) { - console.warn(`[ ${'catalog'.padEnd(9)}] product-data failed (${(error as Error).message}); continuing without it`); + console.warn(`[ ${'catalog'.padEnd(9)}] product-data failed (${(error as Error).message}); published funds are kept`); return new Map(); } })(); console.log(`[ ${'catalog'.padEnd(9)}] ${catalog.length} SPDR ETFs (${catalogSource})`); + // A failed or empty bulk feed is a failed required source for every published fund: its ISIN, CUSIP, SEC yield, + // official dividend yield, net TER and distributions would be rewritten as null or degraded. Such funds keep + // their published state and report failed; only new funds (no meta.json yet) are built. + const bulkFailures = [ + ...(distributions.size ? [] : ['dividend-distribution feed']), + ...(productData.size ? [] : ['product-data workbook']), + ]; + const fileExists = (path: string): Promise => access(path).then(() => true, () => false); + const metaPath = (ticker: string): string => decodeURIComponent(new URL(`funds/${ticker}/meta.json`, API_ROOT).pathname); // A closed fund whose row is published is a known ticker even after the live catalog dropped it (the run then writes only its closed mark). const publishedTickers = new Set((previousIndex?.funds || []).map((row: JsonRecord) => String(row.ticker))); @@ -1751,7 +1765,9 @@ export async function main(env: Record = process.env if (index >= batch.length) return; const fund = batch[index]; const before = await output.before(fund.ticker); - const result = await processFund(fund, distributions.get(fund.ticker), productData.get(fund.ticker), config); + const result: FundResult = bulkFailures.length && (await fileExists(metaPath(fund.ticker))) + ? { ticker: fund.ticker, status: 'failed', reason: `${bulkFailures.join(' and ')} failed, kept as published`, changed: false } + : await processFund(fund, distributions.get(fund.ticker), productData.get(fund.ticker), config); results.push(result); await output.result(fund.ticker, before, result.status === 'failed' || result.status === 'skipped' ? result.status : undefined, result.reason); } @@ -1762,8 +1778,6 @@ export async function main(env: Record = process.env // Live details from disk for refreshed funds. const refreshed = new Set(results.filter((result) => result.status === 'updated' || result.status === 'unchanged').map((result) => result.ticker)); - const fileExists = (path: string): Promise => access(path).then(() => true, () => false); - const metaPath = (ticker: string): string => decodeURIComponent(new URL(`funds/${ticker}/meta.json`, API_ROOT).pathname); const counts = new Map(); for (const ticker of refreshed) { try {