Skip to content

style(output): unify console output with the sibling updaters #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daggerok merged 3 commits into from
Oct 13, 2026
Merged

style(output): unify console output with the sibling updaters #41

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19ba735
style(output): unify console output with the sibling updaters
daggerok Oct 13, 2026
7778481
refactor(output): use the one shared console output block
daggerok Oct 13, 2026
910a7c5
style(output): treat TICKERS as a selection, print 1 fund in the sing…
daggerok Oct 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 49 additions & 17 deletions scripts/update-data.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,12 @@ function outputConfigEntries(config: Record<string, any>): [string, string][] {
performanceRanges: 'PERFORMANCE', totalReturnRanges: 'TOTAL_RETURN',
skipVanEck: 'SKIP_VANECK', skipProShares: 'SKIP_PROSHARES',
skipWisdomTree: 'SKIP_WISDOMTREE', skipGoldmanSachs: 'SKIP_GOLDMANSACHS',
skipGlobalX: 'SKIP_GLOBALX', skipNorthernTrust: 'SKIP_NORTHERNTRUST',
};
const range = (v: any): string => v?.source ?? `${Number.isFinite(v?.min) ? v.min : ''}:${Number.isFinite(v?.max) ? v.max : ''}`;
const range = (v: any): string => typeof v === 'string' ? v : v?.source ?? `${Number.isFinite(v?.min) ? v.min : ''}:${Number.isFinite(v?.max) ? v.max : ''}`;
for (const [key, value] of Object.entries(config)) {
const name = aliases[key] ?? key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toUpperCase();
if (['proxyGapMs', 'fetchTimeoutMs', 'retryDelayMs'].includes(key)) continue;
const name = aliases[key] ?? (/^[A-Z0-9_]+$/.test(key) ? key : key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toUpperCase());
if (name === 'PERFORMANCE' || name === 'TOTAL_RETURN') {
for (const period of ['YTD', '1Y', '3Y', '5Y', '10Y']) values.set(`${name}_${period}`, range(value?.[period]));
} else if (['AUM', 'TER', 'DIVIDEND_YIELD', 'SEC_YIELD'].includes(name)) {
Expand All @@ -80,13 +82,18 @@ function outputConfigEntries(config: Record<string, any>): [string, string][] {
return (ai < 0 ? first.length : ai) - (bi < 0 ? first.length : bi) || a.localeCompare(b);
});
}
function outputConfigValue(key: string, value: string): string {
if (/TOKEN|PASSWORD|SECRET|COOKIE|SEC_UA/i.test(key)) return '<redacted>';
if (key === 'TICKERS' && ['', 'null', 'undefined'].includes(value)) return 'all';
return outputClean(value);
}
function outputPrintConfig(brand: string, config: Record<string, any>): void {
const entries: [string, string][] = [...outputConfigEntries(config), ['VERBOSE', String(outputVerbose())]];
console.log(`[ config ] ${brand} updater:\n${entries.map(([key, value]) => ` ${key}=${/TOKEN|PASSWORD|SECRET|COOKIE|^SEC_UA$/i.test(key) ? '<redacted>' : outputClean(value)}`).join('\n')}`);
console.log(`[ config ] ${brand} updater:\n${entries.map(([key, value]) => ` ${key}=${outputConfigValue(key, value)}`).join('\n')}`);
}
function outputHasOutputFilters(config: Record<string, any>): boolean {
return outputConfigEntries(config).some(([name, value]) =>
/^(TICKERS|CATEGORY|AUM|TER|DIVIDEND_YIELD|SEC_YIELD|PERFORMANCE_|TOTAL_RETURN_)/.test(name) &&
/^(CATEGORY|AUM|TER|DIVIDEND_YIELD|SEC_YIELD|PERFORMANCE_|TOTAL_RETURN_)/.test(name) &&
!['', ':', 'null', 'all'].includes(value));
}
function outputPrintFilter(selected: number, total: number, deferred = false): void {
Expand Down Expand Up @@ -118,19 +125,30 @@ async function outputInspectFund(root: URL | string, ticker: string): Promise<{
}
const outputCount = (value: any): unknown => typeof value === 'number' ? value : Array.isArray(value) ? value.length : value?.totalRows ?? value?.rows?.length ?? null;
const outputScalar = (value: any): any => value && typeof value === 'object' ? value.display ?? value.value ?? null : value;
const outputUnits: Record<string, number> = { K: 1e3, M: 1e6, B: 1e9, T: 1e12, THOUSAND: 1e3, MILLION: 1e6, BILLION: 1e9, TRILLION: 1e12 };
function outputMoney(value: any): string {
const raw = outputScalar(value);
const raw = value && typeof value === 'object' && typeof value.value === 'number' && Number.isFinite(value.value) ? value.value : outputScalar(value);
if (raw === null || raw === undefined || raw === '—' || raw === '--') return 'null';
const text = String(raw).replace(/[$,\s]/g, '');
const match = text.match(/^([+-]?[\d.]+)([KMBT])?$/i);
const match = text.match(/^([+-]?[\d.]+)(K|M|B|T|Thousand|Million|Billion|Trillion)?$/i);
if (!match) return outputClean(raw);
const number = Number(match[1]) * ({ K: 1e3, M: 1e6, B: 1e9, T: 1e12 }[match[2]?.toUpperCase() as 'K' | 'M' | 'B' | 'T'] ?? 1);
const number = Number(match[1]) * (outputUnits[match[2]?.toUpperCase() ?? ''] ?? 1);
if (!Number.isFinite(number)) return 'null';
for (const [unit, scale] of [['T', 1e12], ['B', 1e9], ['M', 1e6], ['K', 1e3]] as const) {
if (Math.abs(number) >= scale) return `$${(number / scale).toFixed(1)}${unit}`;
}
return `$${number.toFixed(2)}`;
}
/** Yields: rounded to 2 decimals without trailing zeros; placeholders are omitted like null. */
function outputPercent(value: any): unknown {
const raw = outputScalar(value);
if (raw === null || raw === undefined) return null;
if (typeof raw === 'number') return Number.isFinite(raw) ? Number(raw.toFixed(2)) : null;
const text = String(raw).trim().replace(/%$/, '').trim();
if (['', '-', '—', '--', 'n/a'].includes(text.toLowerCase())) return null;
const number = Number(text.replace(/,/g, ''));
return Number.isFinite(number) ? Number(number.toFixed(2)) : outputClean(raw);
}
function outputFundLine(index: number, total: number, ticker: string, status: string, data: any = {}, reason?: unknown): string {
const width = Math.max(2, String(total).length);
const metrics = data.metrics ?? {};
Expand All @@ -150,22 +168,35 @@ function outputFundLine(index: number, total: number, ticker: string, status: st
field('divs', outputCount(data.worksheets?.Distributions ?? data.distributions)),
field('netAssets', outputMoney(data.netAssets ?? data.aum)),
field('total', outputMoney(data.totalFundNetAssets ?? data.totalNetAssets)),
field('div', outputScalar(data.trailingYield ?? data.yields?.effectiveYield ?? data.yields?.dividendYield ?? data.dividendYield ?? metrics.dividendYield)),
field('sec', outputScalar(data.secYield ?? data.yields?.secYield ?? metrics.secYield)),
field('div', outputPercent(data.trailingYield ?? data.yields?.effectiveYield ?? data.yields?.dividendYield ?? data.dividendYield ?? metrics.dividendYield)),
field('sec', outputPercent(data.secYield ?? data.yields?.secYield ?? metrics.secYield)),
field('wp', data.workplaceRaw),
].filter(part => part !== '').join(' ');
return `[ ${String(index).padStart(width)}/${String(total).padEnd(width)} ] ${outputClean(ticker).padEnd(5)} ${status.padEnd(9)}${detail ? ` ${detail}` : ''}${reason ? ` reason=${outputClean(reason)}` : ''}`;
}
function outputCreateReporter(root: URL | string, total: number) {
let completed = 0;
const tally = new Map<string, number>();
return {
before: (ticker: string) => outputInspectFund(root, ticker),
async result(ticker: string, before: { digest: string }, status?: string, reason?: unknown, extra: any = {}) {
const after = await outputInspectFund(root, ticker);
console.log(outputFundLine(++completed, total, ticker, status ?? (before.digest === after.digest ? 'unchanged' : 'updated'), { ...after.meta, ...extra }, reason));
const final = status ?? (before.digest === after.digest ? 'unchanged' : 'updated');
tally.set(final, (tally.get(final) ?? 0) + 1);
console.log(outputFundLine(++completed, total, ticker, final, { ...after.meta, ...extra }, reason));
},
/** First [ done ] line: the statuses of every fund line this reporter printed. */
summary(startedAt: number): string {
const order = ['updated', 'unchanged', 'kept', 'skipped', 'failed'];
const statuses = [...order, ...[...tally.keys()].filter(status => !order.includes(status))];
return `[ done ] ${completed} fund${completed === 1 ? '' : 's'}: ${statuses.map(status => `${tally.get(status) ?? 0} ${status}`).join(', ')} in ${((Date.now() - startedAt) / 1000).toFixed(1)}s`;
},
};
}
/** Second [ done ] line: published totals. */
function outputCounts(funds: number, holdings: number, history: number): string {
return `[ done ] counts: ${funds.toLocaleString('en-US')} funds / ${holdings.toLocaleString('en-US')} holdings rows / ${history.toLocaleString('en-US')} history rows`;
}


// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1804,7 +1835,8 @@ export function createRequestClients(config: UpdaterConfig, runtime: RequestRunt
arkProxyActive = true;
if (firstSwitch) {
const notice = `[ ${'issuer'.padEnd(9)}] ARK direct requests returned ${consecutiveArkDenials} consecutive HTTP 403 responses; using the read-only r.jina.ai proxy for remaining issuer requests (shared limit of about 20 requests per minute).`;
(runtime.onIssuerProxy ?? ((message: string) => console.warn(message)))(notice);
// Printed through outputNote so it never lands between fund lines; runUpdater reports the switch before the fund loop.
(runtime.onIssuerProxy ?? outputNote)(notice);
}
return arkProxy(url);
}
Expand Down Expand Up @@ -2897,12 +2929,14 @@ export async function runUpdater(options: RunUpdaterOptions = {}): Promise<RunUp
await rm(stateFile, { force: true });
}

if (clients.isArkProxyActive() && !outputVerbose()) {
console.warn(`[ ${'issuer'.padEnd(9)}] ARK direct requests are denied from this network; using the read-only r.jina.ai proxy for remaining issuer requests`);
}
const reporter = outputCreateReporter(apiRoot, selectedFunds.length);
const secFallback = config.edgarFallback ? createSecFallbackResolver(clients) : undefined;
const processedTickers: string[] = [];
const filteredTickers: string[] = [];
const failedTickers: string[] = [];
let successes = 0;
let nextFundIndex = 0;
let deadlineReached = false;
const worker = async (): Promise<void> => {
Expand Down Expand Up @@ -2943,7 +2977,6 @@ export async function runUpdater(options: RunUpdaterOptions = {}): Promise<RunUp
else if (result.status === 'failed') failedTickers.push(fund.ticker);
else {
indexEntries.set(fund.ticker, result.entry);
successes += 1;
}
await reporter.result(fund.ticker, before, result.status, result.reason, {
...result.entry,
Expand Down Expand Up @@ -2990,10 +3023,9 @@ export async function runUpdater(options: RunUpdaterOptions = {}): Promise<RunUp
scopes[scopeKey] = { cursor: nextCursor ?? 0, tickers: candidates.map((fund) => fund.ticker) };
await writeJsonIfChanged(stateFile, { scopes, generatedAt: isoStamp() });
}
const holdingsTotal = sumIndexCounts(sortedEntries, 'holdings');
const historyTotal = sumIndexCounts(sortedEntries, 'history');
console.log(`[ ${'done'.padEnd(9)}] ${successes} funds updated, ${failedTickers.length} failures`);
console.log(`[ ${'done'.padEnd(9)}] counts: ${sortedEntries.length} catalog funds · ${holdingsTotal} holdings rows · ${historyTotal} history rows${filteredTickers.length ? ` · ${filteredTickers.length} filtered` : ''}`);
console.log(reporter.summary(startedAt));
const counts = indexDocument.counts as { funds: number; holdings: number; history: number };
console.log(outputCounts(counts.funds, counts.holdings, counts.history));

return {
selectedTickers: selectedFunds.map((fund) => fund.ticker),
Expand Down
Loading