From 19ba7354ade8e760dded4c567137143717669159 Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Tue, 13 Oct 2026 01:20:32 -0400 Subject: [PATCH 1/3] style(output): unify console output with the sibling updaters --- scripts/update-data.ts | 57 +++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 52d150c..e632ae1 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -80,9 +80,14 @@ function outputConfigEntries(config: Record): [string, string][] { return (ai < 0 ? first.length : ai) - (bi < 0 ? first.length : bi) || a.localeCompare(b); }); } +function outputConfigValue(key: string, value: string): string { + if (/TOKEN|PASSWORD|SECRET|COOKIE|SEC_UA/i.test(key)) return ''; + if (key === 'TICKERS' && ['', 'null', 'undefined'].includes(value)) return 'all'; + return outputClean(value); +} function outputPrintConfig(brand: string, config: Record): void { const entries: [string, string][] = [...outputConfigEntries(config), ['VERBOSE', String(outputVerbose())]]; - console.log(`[ config ] ${brand} updater:\n${entries.map(([key, value]) => ` ${key}=${/TOKEN|PASSWORD|SECRET|COOKIE|^SEC_UA$/i.test(key) ? '' : outputClean(value)}`).join('\n')}`); + console.log(`[ config ] ${brand} updater:\n${entries.map(([key, value]) => ` ${key}=${outputConfigValue(key, value)}`).join('\n')}`); } function outputHasOutputFilters(config: Record): boolean { return outputConfigEntries(config).some(([name, value]) => @@ -118,19 +123,30 @@ async function outputInspectFund(root: URL | string, ticker: string): Promise<{ } const outputCount = (value: any): unknown => typeof value === 'number' ? value : Array.isArray(value) ? value.length : value?.totalRows ?? value?.rows?.length ?? null; const outputScalar = (value: any): any => value && typeof value === 'object' ? value.display ?? value.value ?? null : value; +const outputUnits: Record = { K: 1e3, M: 1e6, B: 1e9, T: 1e12, THOUSAND: 1e3, MILLION: 1e6, BILLION: 1e9, TRILLION: 1e12 }; function outputMoney(value: any): string { - const raw = outputScalar(value); + const raw = value && typeof value === 'object' && typeof value.value === 'number' && Number.isFinite(value.value) ? value.value : outputScalar(value); if (raw === null || raw === undefined || raw === '—' || raw === '--') return 'null'; const text = String(raw).replace(/[$,\s]/g, ''); - const match = text.match(/^([+-]?[\d.]+)([KMBT])?$/i); + const match = text.match(/^([+-]?[\d.]+)(K|M|B|T|Thousand|Million|Billion|Trillion)?$/i); if (!match) return outputClean(raw); - const number = Number(match[1]) * ({ K: 1e3, M: 1e6, B: 1e9, T: 1e12 }[match[2]?.toUpperCase() as 'K' | 'M' | 'B' | 'T'] ?? 1); + const number = Number(match[1]) * (outputUnits[match[2]?.toUpperCase() ?? ''] ?? 1); if (!Number.isFinite(number)) return 'null'; for (const [unit, scale] of [['T', 1e12], ['B', 1e9], ['M', 1e6], ['K', 1e3]] as const) { if (Math.abs(number) >= scale) return `$${(number / scale).toFixed(1)}${unit}`; } return `$${number.toFixed(2)}`; } +/** Yields: rounded to 2 decimals without trailing zeros; placeholders are omitted like null. */ +function outputPercent(value: any): unknown { + const raw = outputScalar(value); + if (raw === null || raw === undefined) return null; + if (typeof raw === 'number') return Number.isFinite(raw) ? Number(raw.toFixed(2)) : null; + const text = String(raw).trim().replace(/%$/, '').trim(); + if (['', '-', '—', '--', 'n/a'].includes(text.toLowerCase())) return null; + const number = Number(text.replace(/,/g, '')); + return Number.isFinite(number) ? Number(number.toFixed(2)) : outputClean(raw); +} function outputFundLine(index: number, total: number, ticker: string, status: string, data: any = {}, reason?: unknown): string { const width = Math.max(2, String(total).length); const metrics = data.metrics ?? {}; @@ -150,22 +166,35 @@ function outputFundLine(index: number, total: number, ticker: string, status: st field('divs', outputCount(data.worksheets?.Distributions ?? data.distributions)), field('netAssets', outputMoney(data.netAssets ?? data.aum)), field('total', outputMoney(data.totalFundNetAssets ?? data.totalNetAssets)), - field('div', outputScalar(data.trailingYield ?? data.yields?.effectiveYield ?? data.yields?.dividendYield ?? data.dividendYield ?? metrics.dividendYield)), - field('sec', outputScalar(data.secYield ?? data.yields?.secYield ?? metrics.secYield)), + field('div', outputPercent(data.trailingYield ?? data.yields?.effectiveYield ?? data.yields?.dividendYield ?? data.dividendYield ?? metrics.dividendYield)), + field('sec', outputPercent(data.secYield ?? data.yields?.secYield ?? metrics.secYield)), field('wp', data.workplaceRaw), ].filter(part => part !== '').join(' '); return `[ ${String(index).padStart(width)}/${String(total).padEnd(width)} ] ${outputClean(ticker).padEnd(5)} ${status.padEnd(9)}${detail ? ` ${detail}` : ''}${reason ? ` reason=${outputClean(reason)}` : ''}`; } function outputCreateReporter(root: URL | string, total: number) { let completed = 0; + const tally = new Map(); return { before: (ticker: string) => outputInspectFund(root, ticker), async result(ticker: string, before: { digest: string }, status?: string, reason?: unknown, extra: any = {}) { const after = await outputInspectFund(root, ticker); - console.log(outputFundLine(++completed, total, ticker, status ?? (before.digest === after.digest ? 'unchanged' : 'updated'), { ...after.meta, ...extra }, reason)); + const final = status ?? (before.digest === after.digest ? 'unchanged' : 'updated'); + tally.set(final, (tally.get(final) ?? 0) + 1); + console.log(outputFundLine(++completed, total, ticker, final, { ...after.meta, ...extra }, reason)); + }, + /** First [ done ] line: the statuses of every fund line this reporter printed. */ + summary(startedAt: number): string { + const order = ['updated', 'unchanged', 'kept', 'skipped', 'failed']; + const statuses = [...order, ...[...tally.keys()].filter(status => !order.includes(status))]; + return `[ done ] ${completed} funds: ${statuses.map(status => `${tally.get(status) ?? 0} ${status}`).join(', ')} in ${((Date.now() - startedAt) / 1000).toFixed(1)}s`; }, }; } +/** Second [ done ] line: published totals. */ +function outputCounts(funds: number, holdings: number, history: number): string { + return `[ done ] counts: ${funds.toLocaleString('en-US')} funds / ${holdings.toLocaleString('en-US')} holdings rows / ${history.toLocaleString('en-US')} history rows`; +} // --------------------------------------------------------------------------- @@ -1804,7 +1833,8 @@ export function createRequestClients(config: UpdaterConfig, runtime: RequestRunt arkProxyActive = true; if (firstSwitch) { const notice = `[ ${'issuer'.padEnd(9)}] ARK direct requests returned ${consecutiveArkDenials} consecutive HTTP 403 responses; using the read-only r.jina.ai proxy for remaining issuer requests (shared limit of about 20 requests per minute).`; - (runtime.onIssuerProxy ?? ((message: string) => console.warn(message)))(notice); + // Printed through outputNote so it never lands between fund lines; runUpdater reports the switch before the fund loop. + (runtime.onIssuerProxy ?? outputNote)(notice); } return arkProxy(url); } @@ -2897,12 +2927,14 @@ export async function runUpdater(options: RunUpdaterOptions = {}): Promise => { @@ -2943,7 +2975,6 @@ export async function runUpdater(options: RunUpdaterOptions = {}): Promise fund.ticker) }; await writeJsonIfChanged(stateFile, { scopes, generatedAt: isoStamp() }); } - const holdingsTotal = sumIndexCounts(sortedEntries, 'holdings'); - const historyTotal = sumIndexCounts(sortedEntries, 'history'); - console.log(`[ ${'done'.padEnd(9)}] ${successes} funds updated, ${failedTickers.length} failures`); - console.log(`[ ${'done'.padEnd(9)}] counts: ${sortedEntries.length} catalog funds · ${holdingsTotal} holdings rows · ${historyTotal} history rows${filteredTickers.length ? ` · ${filteredTickers.length} filtered` : ''}`); + console.log(reporter.summary(startedAt)); + console.log(outputCounts(indexDocument.counts.funds, indexDocument.counts.holdings, indexDocument.counts.history)); return { selectedTickers: selectedFunds.map((fund) => fund.ticker), From 77784816c53c36a642f8751f1513bcba10a68d1e Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Tue, 13 Oct 2026 01:34:21 -0400 Subject: [PATCH 2/3] refactor(output): use the one shared console output block The output* block is now textually identical across the brand updaters (superset of the per-repo alias maps). --- scripts/update-data.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/update-data.ts b/scripts/update-data.ts index e632ae1..8e69983 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -62,10 +62,12 @@ function outputConfigEntries(config: Record): [string, string][] { performanceRanges: 'PERFORMANCE', totalReturnRanges: 'TOTAL_RETURN', skipVanEck: 'SKIP_VANECK', skipProShares: 'SKIP_PROSHARES', skipWisdomTree: 'SKIP_WISDOMTREE', skipGoldmanSachs: 'SKIP_GOLDMANSACHS', + skipGlobalX: 'SKIP_GLOBALX', skipNorthernTrust: 'SKIP_NORTHERNTRUST', }; - const range = (v: any): string => v?.source ?? `${Number.isFinite(v?.min) ? v.min : ''}:${Number.isFinite(v?.max) ? v.max : ''}`; + const range = (v: any): string => typeof v === 'string' ? v : v?.source ?? `${Number.isFinite(v?.min) ? v.min : ''}:${Number.isFinite(v?.max) ? v.max : ''}`; for (const [key, value] of Object.entries(config)) { - const name = aliases[key] ?? key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toUpperCase(); + if (['proxyGapMs', 'fetchTimeoutMs', 'retryDelayMs'].includes(key)) continue; + const name = aliases[key] ?? (/^[A-Z0-9_]+$/.test(key) ? key : key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toUpperCase()); if (name === 'PERFORMANCE' || name === 'TOTAL_RETURN') { for (const period of ['YTD', '1Y', '3Y', '5Y', '10Y']) values.set(`${name}_${period}`, range(value?.[period])); } else if (['AUM', 'TER', 'DIVIDEND_YIELD', 'SEC_YIELD'].includes(name)) { @@ -3022,7 +3024,8 @@ export async function runUpdater(options: RunUpdaterOptions = {}): Promise fund.ticker), From 910a7c5244fae9c26238cdac11ba027ba1bc1333 Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Tue, 13 Oct 2026 01:40:25 -0400 Subject: [PATCH 3/3] style(output): treat TICKERS as a selection, print 1 fund in the singular --- scripts/update-data.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 8e69983..4ff286a 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -93,7 +93,7 @@ function outputPrintConfig(brand: string, config: Record): void { } function outputHasOutputFilters(config: Record): boolean { return outputConfigEntries(config).some(([name, value]) => - /^(TICKERS|CATEGORY|AUM|TER|DIVIDEND_YIELD|SEC_YIELD|PERFORMANCE_|TOTAL_RETURN_)/.test(name) && + /^(CATEGORY|AUM|TER|DIVIDEND_YIELD|SEC_YIELD|PERFORMANCE_|TOTAL_RETURN_)/.test(name) && !['', ':', 'null', 'all'].includes(value)); } function outputPrintFilter(selected: number, total: number, deferred = false): void { @@ -189,7 +189,7 @@ function outputCreateReporter(root: URL | string, total: number) { summary(startedAt: number): string { const order = ['updated', 'unchanged', 'kept', 'skipped', 'failed']; const statuses = [...order, ...[...tally.keys()].filter(status => !order.includes(status))]; - return `[ done ] ${completed} funds: ${statuses.map(status => `${tally.get(status) ?? 0} ${status}`).join(', ')} in ${((Date.now() - startedAt) / 1000).toFixed(1)}s`; + return `[ done ] ${completed} fund${completed === 1 ? '' : 's'}: ${statuses.map(status => `${tally.get(status) ?? 0} ${status}`).join(', ')} in ${((Date.now() - startedAt) / 1000).toFixed(1)}s`; }, }; }