diff --git a/backend/.env.dist.composed b/backend/.env.dist.composed index b0af8d7a05..efb78e0a52 100644 --- a/backend/.env.dist.composed +++ b/backend/.env.dist.composed @@ -36,5 +36,8 @@ CROWD_PACKAGES_DB_USERNAME=postgres CROWD_PACKAGES_DB_PASSWORD=example CROWD_PACKAGES_DB_DATABASE=packages-db +# Packagist registry crawler contact email +CROWD_PACKAGES_PACKAGIST_MAILTO= + # security-contacts-worker SECURITY_CONTACTS_USER_AGENT="lfx-security-contacts-worker" diff --git a/backend/.env.dist.local b/backend/.env.dist.local index 3e9c73b85d..8286091987 100755 --- a/backend/.env.dist.local +++ b/backend/.env.dist.local @@ -180,6 +180,9 @@ CROWD_PACKAGES_DB_USERNAME=postgres CROWD_PACKAGES_DB_PASSWORD=example CROWD_PACKAGES_DB_DATABASE=packages-db +# Packagist registry crawler contact email +CROWD_PACKAGES_PACKAGIST_MAILTO= + # github-repos-enricher ENRICHER_GITHUB_TOKENS= ENRICHER_BATCH_SIZE=100 diff --git a/backend/src/osspckgs/migrations/V1784314023__packagist_worker.sql b/backend/src/osspckgs/migrations/V1784314023__packagist_worker.sql new file mode 100644 index 0000000000..2d8a2e1d5b --- /dev/null +++ b/backend/src/osspckgs/migrations/V1784314023__packagist_worker.sql @@ -0,0 +1,17 @@ + +-- Packagist worker: per-lane ingestion state tracking (metadata, downloads-30d, daily downloads) +CREATE TABLE packagist_package_state ( + purl text PRIMARY KEY, + first_seen_at timestamptz NOT NULL DEFAULT now(), + metadata_last_run_at timestamptz, + metadata_run_result jsonb, -- { status, attempts, httpStatus?, errorKind?, message? } + metadata_last_modified text, -- Last-Modified from p2 endpoint, replayed as If-Modified-Since + downloads_30d_last_run_at timestamptz, + downloads_30d_run_result jsonb, -- { status, attempts, httpStatus?, errorKind?, message? } + daily_downloads_last_run_at timestamptz, + daily_downloads_run_result jsonb -- { status, attempts, httpStatus?, errorKind?, message? } +); + +CREATE INDEX ON packagist_package_state (metadata_last_run_at); +CREATE INDEX ON packagist_package_state (downloads_30d_last_run_at); +CREATE INDEX ON packagist_package_state (daily_downloads_last_run_at); diff --git a/docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md b/docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md index 6089d04062..fc2d530671 100644 --- a/docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md +++ b/docs/adr/0005-pypi-downloads-bigquery-merge-scoping.md @@ -2,7 +2,7 @@ **Date**: 2026-07-01 **Status**: accepted -**Deciders**: Anil B +**Deciders**: Anil Bostanci _Consolidated ADR for the PyPI downloads worker — record further PyPI-worker download decisions here rather than opening new ADRs._ diff --git a/docs/adr/0009-packagist-worker-design-decisions.md b/docs/adr/0009-packagist-worker-design-decisions.md new file mode 100644 index 0000000000..5d77f0b932 --- /dev/null +++ b/docs/adr/0009-packagist-worker-design-decisions.md @@ -0,0 +1,200 @@ +# ADR-0009: Packagist worker — design decisions (living) + +**Date**: 2026-07-13 +**Status**: accepted +**Deciders**: Anil Bostanci + +_Living, consolidated record for the Packagist (PHP/Composer) worker. Record further +Packagist-worker decisions here as new `### subsections` under Decisions rather than opening a +new ADR per decision — mirrors ADR-0001 (oss-packages living ADR) and the ADR-0005 consolidation +note. Each entry is stamped with its own `**Decided**` date; the Changelog at the bottom tracks +when entries are added or revised._ + +## Context + +Packagist is the primary registry for PHP/Composer. Unlike npm/maven/pypi/etc., it is **not +covered by deps.dev** — there is no `PACKAGIST` system in `bigquery-public-data.deps_dev_v1` — so we +cannot seed it from BigQuery the way ADR-0001's universe ingest does for other ecosystems. We crawl +Packagist directly instead (list.json seed → dynamic stats endpoint → static p2 metadata endpoint). +A consequence that shapes several decisions: Packagist hands us **per-package Composer manifests** +(the raw `require` / `require-dev` maps a package declared), **not a pre-resolved dependency graph** +like the one deps.dev exposes in BigQuery. + +## Scope and current status + +| Decision area | Status | +| --------------- | ------- | +| Dependency model | decided | +| Metadata enrichment scope | decided | +| Lane architecture & cadence | decided | + +--- + +## Decisions + +### Dependency model: direct edges + declared constraints, resolved at query time + +We store, in `package_dependencies`, **only the direct dependency edges a version declares** — one +row per `(version_id, depends_on_id, dependency_kind)` — and we **resolve concrete versions and +transitive closure at query time**, never at ingest. + +Per stored edge (for **all** packages — dependency edges ride the same p2 metadata fetch as +versions; see the Metadata enrichment scope decision): + +- `require` → `dependency_kind = 'direct'`, `require-dev` → `dependency_kind = 'dev'`. +- `version_constraint` = the **declared** constraint string verbatim (e.g. `^3.0 || ^2.0`). +- `depends_on_id` = the depended-on **package** row (resolved by name; edges whose target isn't a + known packages row are counted and skipped, not errored). +- `depends_on_version_id` = **NULL** — we do not resolve the concrete satisfying version at ingest. +- Platform requirements (`php`, `hhvm`, `ext-*`, `lib-*`, `composer-*` — anything with no `/`) are + excluded; they aren't packages. + +**Provenance.** The query-time transitive model is a **stated requirement**, quoted verbatim from +the Packagist change brief: + +> We store only direct dependencies (the requirements declared in each version's composer.json) and +> walk the tree at query time when transitive resolution is needed. We do not need to compute or +> store the resolved graph at ingest. + +Leaving `depends_on_version_id` NULL is the column-level **implementation** of that last sentence — +the resolved concrete version *is* part of "the resolved graph" — locked at the plan-approval gate +and consistent with the pre-existing schema comment (`depends_on_version_id bigint, -- resolved +version; NULL if unknown`). Storing the declared `version_constraint` but not a pinned version means +the two derived views — *which version a constraint resolves to* and *the full transitive tree* — +are both computed on read. + +**Why not pin `depends_on_version_id` at ingest (as deps.dev does):** a resolved version is *derived +and volatile*. `^3.0` resolves to whatever `psr/log` patch is newest this week; a stored pin goes +stale the moment a patch ships and would need continuous rewriting. deps.dev's own worker documents +exactly this churn — re-resolving `^4.17.0` every snapshot is what produced its ~555M-row edge +exports and forced a snapshot-diff to suppress it. We sidestep it entirely by storing the immutable +declared constraint and resolving on demand. deps.dev pins because it *already* has a fully-resolved +graph in BigQuery for free; we only get manifests, so resolving eagerly would mean building and +continuously re-running a Composer resolver for no durable benefit. + +**Why store `dev` edges when deps.dev doesn't:** deps.dev's resolved graph is runtime-only — every +edge lands as `'direct'`, it has no dev/peer notion. Because we read the raw manifest we can split +`require-dev` into `'dev'`, so Packagist edges are *richer per-edge* (dev deps + declared +constraints) even though they are *shallower* (no resolved target version) than deps.dev's. + +**Consequences.** + +_Positive:_ + +- Write path is small and touches only immutable declared facts — no re-resolution churn, no + billion-row resolved-graph materialization for Packagist. +- Query-time resolution and transitive walks always reflect current data (new patches, newly-added + deep dependencies) with no re-ingest. +- Captures dev-dependencies (`kind='dev'`), a dimension deps.dev-seeded ecosystems lack. +- Schema-compatible with the deps.dev-populated `package_dependencies`: same table, same unique key + `(version_id, depends_on_id, dependency_kind)`; the `depends_on_id` HASH-partitioning (ADR-0001) + keeps the upstream "who depends on X?" hot path fast for Packagist edges too. + +_Negative / trade-offs:_ + +- `depends_on_version_id` is NULL for every Packagist edge — a consumer that wants the exact version + a dependency pulls in must resolve the constraint itself; there is no per-edge pinned version the + way there is for npm/maven/pypi/cargo seeded from deps.dev. +- Transitive questions ("everything X depends on") pay a recursive walk over direct edges at read + time rather than a single indexed lookup. + +_Risks:_ + +- If an LFX product later needs per-edge **resolved** versions for Packagist, that is a + **requirements change** — it reopens the "do not store the resolved graph at ingest" decision and + would require standing up a Composer-style resolver (and accepting the re-resolution churn deps.dev + fights). Flagged here so the trade-off is revisited deliberately, not silently. + +**Decided**: 2026-07-13 + +--- + +### Metadata enrichment scope: all packages, not the critical slice + +The metadata lane (p2 manifests → `versions`, `package_dependencies` edges, and the package-level +aggregates `licenses` / `latest_version` / `versions_count` / `first_release_at` / +`latest_release_at` / `homepage`) runs for **all ~500K Packagist packages**, not only the critical +slice: `CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL` defaults to `false` (setting it `true` +narrows back to the critical slice). The same lane additionally links `repos` / +`package_repos` rows for all packages (`'declared'` @ `0.8`, the manifest-declared convention +shared with npm/pypi/maven/cargo), so the shared github-repos-enricher picks Packagist repos up. + +**Provenance.** Requested in the deps.dev-parity review (Joana): deps.dev has zero Packagist data, +so unlike every other ecosystem there is no external source backfilling the non-critical majority — +the tiering left most of the catalog thin for no data-availability reason. The goal is best-effort +parity with what deps.dev populates for its ecosystems. + +**Why this deliberately diverges from pypi:** the pypi sibling documents critical-only as the +"intended steady state" with all-packages as a temporary bootstrap mode. That design assumes +deps.dev provides package-level data for the whole ecosystem, so per-package registry enrichment +only needs to deepen the critical slice. Packagist has no such backstop; the registry crawl *is* +the universe source. The p2 endpoint is a static, CDN-served file designed to be mirrored (it is +how Composer clients resolve), with `If-Modified-Since`/304 replay — so the steady-state weekly +cost after the first full pass is dominated by 304s, not re-parses. + +**What stays critical-only (deliberate, not parity gaps):** `downloads_daily` and +maintainers. deps.dev carries no downloads at all, and npm/pypi both scope daily downloads and +maintainer enrichment to the critical slice; ungating maintainers would also cost ~3–4M per-row +DB round-trips per weekly cycle (`upsertPackageMaintainers` is 2M+2 queries per package) — a +batching rewrite is a prerequisite before that scope can ever widen. + +**Consequences.** + +_Positive:_ versions, direct dependency edges, licenses/latest-version/release-date aggregates, +homepage, and repo links exist for the whole catalog — matching deps.dev's coverage shape; ranking +inputs and downstream consumers see a fully-populated ecosystem rather than a thin one. + +_Negative / trade-offs:_ the weekly metadata drain walks ~500K purls (~10k keyset batches, ~500 +continueAsNew generations) instead of ~56K; `versions` grows by an estimated 5–10M rows and +`package_dependencies` by ~20–40M edges (both well within the tables' 90M+/1.15B+ design points); +the two packagist workers now default to opposite scopes, mitigated by explicit comments at both +flip points. + +_Risks:_ the all-packages due-selection abandons the partial `is_critical` index and rides the +`purl` btree keyset walk — late-cycle batches skip progressively more already-scanned rows; if the +tail slows measurably, revisit with an `EXPLAIN` (flagged in the rollout checklist). + +**Decided**: 2026-07-16 + +--- + +### Lane architecture: seed → chained enrichment, downloads as dedicated lanes + +Four workflows instead of the earlier seed/stats/metadata split (the `ingestPackagistStats` +workflow, whose name and mixed responsibilities were unclear, was removed): + +1. **`seedPackagistPackages`** — weekly cron. Discovery only; on completion **chain-starts the + enrichment drain as a child workflow** (`ParentClosePolicy.ABANDON`) instead of a second cron, + so "after seed" is an event, not a clock offset (same idiom as `bootstrapOsspckgs`). +2. **`ingestPackagistMetadata`** — the merged enrichment lane: per package it fetches **both** + registry endpoints — dynamic (package info, counters, repo link, maintainers) and p2 (versions, + dependencies, aggregates, homepage) — under one `metadata_last_run_at` watermark. +3. **`ingestPackagistDownloads30d`** — monthly cron **on the 1st**: captures the observed rolling + 30d value as the month's `downloads_last_30d` window row (mirrored to the packages column), + npm's breadth pattern (run-scoped cutoff, per-purl watermark). Running on the boundary replaces + the earlier "first scan on/after the 1st" heuristic. No history/backfill lane — Packagist keeps + no historical series, so a missed month is unrecoverable by design. +4. **`ingestPackagistDownloadsDaily`** — daily cron, critical slice only. + +Each lane owns its own watermark + run result in `packagist_package_state` +(`metadata_* / downloads_30d_* / daily_downloads_*`; the ambiguous `p2_last_modified` renamed to +`metadata_last_modified`). Trade-offs: the enrichment lane makes two HTTP calls per package +(bounded by the dynamic endpoint's 10-concurrency), the dynamic endpoint is fetched by three lanes +at different cadences (~500K extra fetches/month for the 30d lane), and a hard seed failure skips +that week's enrichment (recoverable via the manual trigger). + +**Decided**: 2026-07-16 + +--- + +## Changelog + +- **2026-07-13** — ADR created. First entry: _Dependency model: direct edges + declared constraints, + resolved at query time_. +- **2026-07-16** — Added _Metadata enrichment scope: all packages, not the critical slice_ (also + covers repo linking for all packages and the deliberate critical-only carve-outs). +- **2026-07-16** — Added _Lane architecture: seed → chained enrichment, downloads as dedicated + lanes_ (removes the stats lane; renames `p2_last_modified` → `metadata_last_modified`). +- **2026-07-17** — Corrected stale "critical slice only" wording left over in the _Dependency + model_ entry from before the _Metadata enrichment scope_ decision widened it to all packages; + set `Status` to `accepted` (matching ADR-0005's precedent for a living/consolidated doc). diff --git a/docs/adr/README.md b/docs/adr/README.md index 1aec4e9273..40a8edab4b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -15,6 +15,7 @@ Use the `/adr` skill in Claude Code to record new ADRs or query past decisions. | [ADR-0006](./0006-database-schema-types-as-source-of-truth.md) | Database schema types as the source of truth | accepted | 2026-07-09 | | [ADR-0007](./0007-test-factory-primitives-and-defaults.md) | Test factory primitives and defaults | accepted | 2026-07-10 | | [ADR-0008](./0008-how-we-write-unit-tests.md) | How we write unit tests | accepted | 2026-07-13 | +| [ADR-0009](./0009-packagist-worker-design-decisions.md) | Packagist worker — design decisions | accepted | 2026-07-13 | ## Why ADRs? diff --git a/scripts/builders/packages.env b/scripts/builders/packages.env index 03a4f019dc..692c59f4c3 100644 --- a/scripts/builders/packages.env +++ b/scripts/builders/packages.env @@ -1,4 +1,4 @@ DOCKERFILE="./services/docker/Dockerfile.packages" CONTEXT="../" REPO="sjc.ocir.io/axbydjxa5zuh/packages" -SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker rubygems-worker blast-radius-worker" +SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker rubygems-worker blast-radius-worker packagist-worker" diff --git a/scripts/cli b/scripts/cli index 7bdfd7e3be..f4a18a92db 100755 --- a/scripts/cli +++ b/scripts/cli @@ -1182,14 +1182,14 @@ while test $# -gt 0; do exit ;; service-restart-fe-dev) - IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") + IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "packagist-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") DEV=1 kill_all_containers service_start exit ;; clean-start-fe-dev) - IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") + IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker" "automatic-projects-discovery-worker" "pcc-sync-worker" "projects-evaluation-worker" "bq-dataset-ingest" "cargo-worker" "dockerhub-sync" "github-repos-enricher" "go-worker" "maven-worker" "npm-worker" "nuget-worker" "osv-worker" "packagist-worker" "pypi-worker" "rubygems-worker" "security-contacts-worker") CLEAN_START=1 DEV=1 start diff --git a/scripts/services/packagist-worker.yaml b/scripts/services/packagist-worker.yaml new file mode 100644 index 0000000000..8e7ea99b8c --- /dev/null +++ b/scripts/services/packagist-worker.yaml @@ -0,0 +1,70 @@ +version: '3.1' + +x-env-args: &env-args + DOCKER_BUILDKIT: 1 + NODE_ENV: docker + SERVICE: packagist-worker + CROWD_TEMPORAL_TASKQUEUE: packagist-worker + CROWD_TEMPORAL_NAMESPACE: ${CROWD_PACKAGES_TEMPORAL_NAMESPACE} + SHELL: /bin/sh + SUPPRESS_NO_CONFIG_WARNING: 'true' + +services: + packagist-worker: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.packages + command: 'pnpm run start:packagist-worker' + working_dir: /usr/crowd/app/services/apps/packages_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + restart: always + networks: + - crowd-bridge + + packagist-worker-dev: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.packages + command: 'pnpm run dev:packagist-worker' + working_dir: /usr/crowd/app/services/apps/packages_worker + # user: '${USER_ID}:${GROUP_ID}' + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + dns: + - 8.8.8.8 + - 1.1.1.1 + hostname: packagist-worker + networks: + - crowd-bridge + volumes: + - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src + - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src + - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src + - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src + - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src + - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src + - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src + - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src + - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src + - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src + - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src + - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src + - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src + - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src + - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src + - ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src + +networks: + crowd-bridge: + external: true diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index e4deb21964..825d0adf18 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -25,6 +25,8 @@ "trigger-go:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=go-worker tsx src/scripts/triggerGoEnrich.ts", "trigger-nuget": "SERVICE=nuget-worker tsx src/scripts/triggerNuGetSync.ts", "trigger-nuget:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=nuget-worker tsx src/scripts/triggerNuGetSync.ts", + "trigger-packagist": "SERVICE=packagist-worker tsx src/scripts/triggerPackagistSeed.ts", + "trigger-packagist:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=packagist-worker tsx src/scripts/triggerPackagistSeed.ts", "start:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker tsx src/bin/npm-worker.ts", "dev:npm-worker": "CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts", "dev:npm-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=npm-worker SERVICE=npm-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9236 src/bin/npm-worker.ts", @@ -49,6 +51,9 @@ "start:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker tsx src/bin/nuget-worker.ts", "dev:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts", + "start:packagist-worker": "CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker tsx src/bin/packagist-worker.ts", + "dev:packagist-worker": "CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9245 src/bin/packagist-worker.ts", + "dev:packagist-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packagist-worker SERVICE=packagist-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9245 src/bin/packagist-worker.ts", "start:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker tsx src/bin/rubygems-worker.ts", "dev:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts", "dev:rubygems-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts", diff --git a/services/apps/packages_worker/src/activities.ts b/services/apps/packages_worker/src/activities.ts index 0e3b4a5738..25b3c1a365 100644 --- a/services/apps/packages_worker/src/activities.ts +++ b/services/apps/packages_worker/src/activities.ts @@ -33,6 +33,21 @@ export { } from './pypi/activities' export { getCriticalPypiCount } from './pypi/downloads/getCriticalPypiCount' export { processNuGetBatch } from './nuget/activities' +export { + ingestOnePackagistMetadata, + ingestOnePackagist30dWindow, + ingestOnePackagistDailyDownload, + runPackagistPackageSeed, + getPackagistMetadataBatch, + ingestPackagistMetadataBatch, + getPackagist30dBatch, + ingestPackagist30dBatch, + getPackagistDailyBatch, + ingestPackagistDailyBatch, + getCriticalPackagistCount, + packagistCurrentTimestamp, + packagistStopAfterFirstPage, +} from './packagist/activities' export { processRubyGemsCoreBatch, processRubyGemsCriticalBatch } from './rubygems/activities' export { processSecurityContactsBatch, diff --git a/services/apps/packages_worker/src/bin/packagist-worker.ts b/services/apps/packages_worker/src/bin/packagist-worker.ts new file mode 100644 index 0000000000..0e7e23670a --- /dev/null +++ b/services/apps/packages_worker/src/bin/packagist-worker.ts @@ -0,0 +1,8 @@ +import { schedulePackagistIngest } from '../packagist/schedule' +import { svc } from '../service' + +setImmediate(async () => { + await svc.init() + await schedulePackagistIngest() + await svc.start() +}) diff --git a/services/apps/packages_worker/src/packagist/README.md b/services/apps/packages_worker/src/packagist/README.md new file mode 100644 index 0000000000..29dcec90d2 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/README.md @@ -0,0 +1,212 @@ +# Packagist Worker — Workflows + +The Packagist worker keeps the PHP/Composer slice of the packages database fresh +by crawling **packagist.org directly** — deps.dev has no Packagist coverage, so +unlike npm/maven/pypi there is no BigQuery universe to import from; the registry +crawl _is_ the universe source. It runs **four Temporal workflows** on the +`packagist-worker` task queue: three cron schedules registered at worker boot +(`src/bin/packagist-worker.ts`), plus the metadata drain, which the seed chains +as a child workflow on completion. + +Identity: ecosystem `packagist`, purls `pkg:composer/{vendor}/{name}` +(namespace = vendor). Audit tag: `packagist` in `audit_field_changes`. + +Shared design notes: + +- **Two registry endpoints.** The _dynamic_ endpoint + (`packagist.org/packages/{vendor}/{name}.json`, server-cached ~12h) carries + package info and live counters: downloads, dependents, description, repo URL, + abandoned flag, maintainers. The _static_ p2 endpoint + (`repo.packagist.org/p2/{vendor}/{name}.json`, CDN-served, the same file + Composer clients resolve) carries the version manifests: every tagged + release with its `require`/`require-dev`, licenses, homepage. +- **Politeness.** Bounded concurrency per Packagist's own published etiquette + guideline ([packagist.org/apidoc](https://packagist.org/apidoc)): max 10 + concurrent requests, 20 if static-file-only. Their docs state this as "be a + good citizen," not a documented enforcement mechanism — we don't have a + citation for what happens above it, so treat 10 as the number to respect, + not a proven "or else". Since every lane's ingest starts with a dynamic + fetch, all lanes run at that stricter cap anyway (env-tunable downward, + hard-capped at 10); the p2 fetches inside the metadata lane inherit it, + well under p2's own 20. User-Agent carries a `mailto=` contact + (`CROWD_PACKAGES_PACKAGIST_MAILTO`) — their docs say a missing one risks an + IP block. Cron minutes are deliberately off `:00` (their docs also flag + hourly traffic peaks at `XX:00`). p2 fetches replay the stored + `Last-Modified` as `If-Modified-Since`; a `304` records the run without + re-downloading or re-parsing. +- **Watermark-driven drains.** Each lane tracks its own per-purl watermark + + run result in `packagist_package_state` (`metadata_last_run_at` / + `downloads_30d_last_run_at` / `daily_downloads_last_run_at`, each with a + `*_run_result` JSONB). Workflows drain in `continueAsNew` rounds (20 × 50), + so an interrupted run resumes where it left off on the next launch. +- **Error handling.** 404 or malformed body/shape → 3 fast in-lane retries + (1s linear backoff), then mark the lane's watermark with an error result and + move on — `isClientError()` treats 404 the same as any other non-429 4xx, so + there's no zero-retry fast path for it specifically. 429/5xx/network/timeout + → throw and ride Temporal's exponential activity retry (5 attempts, lockstep + with the per-item give-up so one bad package can never stall a drain). +- **Tier scoping.** Discovery, package info, repo links, versions, and + dependencies cover **all** packages (deliberate divergence from pypi — see + ADR-0009). Only `downloads_daily` and maintainers are + **critical-slice-only**, matching npm/pypi. + +--- + +## 1. `seedPackagistPackages` — universe discovery + +**Schedule:** `packagist-seed`, weekly Sunday `02:17` UTC. +**Targets:** the full catalog (~500K names) — criticality not consulted. + +**What it does** (`runPackagistPackageSeed`): one GET of +`packagist.org/packages/list.json`, validates/lowercases/dedupes the names, +inserts identity rows in 5,000-row chunks, then **chain-starts the metadata +drain** (child workflow, `ParentClosePolicy.ABANDON`). + +**Populates:** `packages` only — `purl`, `ecosystem='packagist'`, +`namespace` (vendor), `name`, `registry_url`, `status='active'`, +`ingestion_source='packagist-registry'`. `ON CONFLICT (purl) DO NOTHING`: +re-runs only add newly published packages and never touch enriched rows. + +--- + +## 2. `ingestPackagistMetadata` — the enrichment drain (both endpoints) + +**Schedule:** none of its own — **chained off the seed** (effectively weekly, +Sunday right after `02:17`), so newly published packages exist as rows before +dependency targets are resolved. Runs independently once started; a failed +seed skips that week's drain (recover with +`pnpm trigger-packagist:local metadata`). +**Targets:** **all** packagist packages due for a refresh — +`CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL` defaults to `false` +(`true` narrows to the critical slice). Due = never scanned or +`metadata_last_run_at` older than 7 days (`…_METADATA_REFRESH_DAYS`). + +**What it does:** per package, **two fetches** — the dynamic endpoint (≤10 +concurrent) and the p2 endpoint (with `If-Modified-Since`; steady state is +mostly 304s). A package the dynamic endpoint 404s is marked errored without +touching p2. + +**Populates, per scanned package:** + +- From the **dynamic endpoint**: + - `packages` — `description`, `declared_repository_url`, `repository_url` + (canonicalized), `status` (`abandoned` → `deprecated`, else `active`), + `total_downloads`, `dependent_count` (registry-reported direct + dependents — a ranking input), `last_synced_at`. Counters COALESCE: a + missing value never wipes a known one. `downloads_last_30d` is + deliberately NOT written here — see the dedicated monthly lane below. + - `repos` + `package_repos` — **all packages**: canonicalized repo URL linked + with `source='declared'`, `confidence=0.8` (the manifest-declared convention + shared with npm/pypi/maven/cargo); feeds the shared github-repos-enricher. + Skipped when no URL or it can't be canonicalized. + - `maintainers` + `package_maintainers` — **critical only**: usernames from + the dynamic endpoint (`role='maintainer'`). +- From the **p2 endpoint**: + - `versions` — one row per tagged release (`number`, `published_at`, + `is_prerelease`, `is_latest`, `licenses`); dev branches skipped. + - `package_dependencies` — direct edges only (`require` → `'direct'`, + `require-dev` → `'dev'`), declared constraints verbatim, platform packages + excluded, `depends_on_version_id` NULL (resolved at query time — ADR-0009). + - `packages` aggregates — `versions_count`, `latest_version`, + `first_release_at`, `latest_release_at`, `licenses`, `homepage` — all + written non-destructively (COALESCE / keep-on-zero). +- `packagist_package_state` — `metadata_last_run_at`, `metadata_run_result`, + and `metadata_last_modified` (the p2 `Last-Modified`, replayed as + `If-Modified-Since` next run). +- `audit_field_changes` — every changed field above, worker tag `packagist`. + +--- + +## 3. `ingestPackagistDownloads30d` — monthly rolling-window capture + +**Schedule:** `packagist-downloads-30d`, monthly on the **1st** at `03:53` UTC — +anchored on the month boundary so the observed rolling-30d value approximates +the calendar month. +**Targets:** **all** packagist packages, npm-style breadth: the run fixes a +`cutoff` timestamp once, and every package whose `downloads_30d_last_run_at` +is older (or null) is due exactly once per run. + +**What it does:** fetches the dynamic endpoint per package and records +`downloads.monthly` as the month's window. + +**Populates:** one `downloads_last_30d` row per purl per month +(`end_date` = the 1st, `start_date` = end − 30d), **mirrored to +`packages.downloads_last_30d`**. A window already recorded for the month is +never overwritten — the first observation (closest to the boundary) wins. +Packagist keeps no history, so unlike npm there is no backfill lane; a missed +month stays missed. State: `downloads_30d_last_run_at` + `downloads_30d_run_result`. + +Like the daily lane (see below), the write-date is pinned to the run's fixed +cutoff and threaded through every batch, so the labeled window stays correct +even if the drain runs long. + +--- + +## 4. `ingestPackagistDownloadsDaily` — daily capture (critical slice) + +**Schedule:** `packagist-downloads-daily`, daily `22:23` UTC — late in the day +on purpose: Packagist's `daily` figure is `today_so_far + yesterday_total × +dayRatio`, with `dayRatio` decaying toward 0 as the day goes on, so asking +late gets a figure that's mostly today's real data instead of mostly +borrowed from yesterday. +**Targets:** `is_critical = TRUE` only; no-ops (guard) while zero packagist +rows are critical — i.e. until the first `rank_packages()` pass. Same cutoff +watermark pattern as the 30d lane. + +**What it does:** fetches the dynamic endpoint per critical package and records +the registry's rolling daily figure. + +**Populates:** one `downloads_daily` row per package per day (`package_id`, +run date, count), with the run date pinned once from the run's `cutoff` so +every row from one run shares the same label even if the drain runs long. +That only fixes the _label_, though — Packagist computes `daily` live at +fetch time, so for a long-running drain the underlying value can drift +forward from what the label implies for whichever packages are processed +last. State: `daily_downloads_last_run_at` + `daily_downloads_run_result`. + +--- + +## What this worker deliberately does NOT write + +- `transitive_dependent_count`, `dependent_repos_count` — not computable from + the registry; needs a reverse-closure over our stored direct edges + (future work, see ADR-0009 risks). +- Advisories — the OSV worker owns security data platform-wide. +- `is_critical` / `criticality_score` / ranking columns — the shared + criticality worker; this worker only _reads_ `is_critical` for scoping. +- Repo stars/forks/scorecard — the shared github-repos-enricher, fed by the + `package_repos` links this worker creates. +- `keywords`, `versions.is_yanked`, `depends_on_version_id` — see above. + +## Configuration + +All optional; defaults in `activities.ts` / `fetchPackage.ts`: + +| Env var | Default | Meaning | +| ------------------------------------------------ | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| `CROWD_PACKAGES_PACKAGIST_MAILTO` | `oss-packages@linuxfoundation.org` | Contact in the crawl User-Agent | +| `CROWD_PACKAGES_PACKAGIST_STATS_CONCURRENCY` | 10 (hard cap 10) | Parallel package ingests across all lanes — every lane starts with a dynamic-endpoint fetch, so the stricter 10-limit bounds everything | +| `CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS` | 7 | Metadata refresh window | +| `CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL` | `false` | `true` narrows metadata to the critical slice | +| `CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE` | `false` | Debug: one drain page, no continueAsNew | + +## Running it + +```bash +# local worker (hot reload; needs packages-db + temporal from scripts/scaffold.yaml) +DEV=1 ./scripts/cli service packagist-worker up + +# trigger on demand instead of waiting for the crons +cd services/apps/packages_worker +pnpm trigger-packagist:local seed # discovery (chain-starts metadata!) +pnpm trigger-packagist:local metadata # enrichment: info + versions + deps +pnpm trigger-packagist:local downloads-30d # monthly rolling-window capture +pnpm trigger-packagist:local downloads-daily # daily capture, critical slice +``` + +Note: triggering `seed` also chain-starts the full `metadata` drain (set +`CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE=true` locally to bound it). +Local smoke order: seed → metadata → (rank) → downloads lanes for +critical-scoped writes. State lives in `packagist_package_state` +(migration `V1784314023__packagist_worker.sql`); design decisions in +`docs/adr/0009-packagist-worker-design-decisions.md`. diff --git a/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts new file mode 100644 index 0000000000..6447b75c90 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/downloads.test.ts @@ -0,0 +1,88 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + insertLast30dDownloadIfAbsent, + logAuditFieldChanges, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { monthlyWindowFor, persistPackagist30dWindow } from '../downloads' + +vi.mock('@crowd/data-access-layer/src/packages', () => ({ + insertLast30dDownloadIfAbsent: vi.fn().mockResolvedValue([]), + logAuditFieldChanges: vi.fn(), +})) + +const mockWindow = vi.mocked(insertLast30dDownloadIfAbsent) +const mockAudit = vi.mocked(logAuditFieldChanges) + +const qx = { + tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), +} as unknown as QueryExecutor +const PURL = 'pkg:composer/monolog/monolog' + +beforeEach(() => { + vi.clearAllMocks() +}) + +// Observed rolling window anchored on the 1st of the run month, +// start_date = end_date − 30 days. +describe('monthlyWindowFor', () => { + it('anchors the window on the 1st of the run month', () => { + expect(monthlyWindowFor('2026-07-15')).toEqual({ + startDate: '2026-06-01', + endDate: '2026-07-01', + }) + expect(monthlyWindowFor('2026-07-01')).toEqual({ + startDate: '2026-06-01', + endDate: '2026-07-01', + }) + }) + + it('computes start_date by calendar arithmetic, not "previous month 1st"', () => { + expect(monthlyWindowFor('2026-03-05')).toEqual({ + startDate: '2026-01-30', + endDate: '2026-03-01', + }) + }) +}) + +// The monthly downloads-30d lane: one window row per purl per month, mirrored to +// packages.downloads_last_30d (npm parity). A window already recorded for the month +// is never overwritten — the value is the observation closest to the boundary. +// insertLast30dDownloadIfAbsent does the presence check and the write atomically, so +// there is no separate "does it exist" call to assert on here — a race is exercised +// directly against Postgres in downloadsLast30d's own DAL-level coverage. The audit +// record shares the same transaction as the write. +describe('persistPackagist30dWindow', () => { + it('writes the window with the mirror, audits it, and returns the changed fields when the month has no row yet', async () => { + mockWindow.mockResolvedValue(['downloads_last_30d.count', 'packages.downloads_last_30d']) + + const changedFields = await persistPackagist30dWindow(qx, PURL, 300, '2026-07-01') + + expect(mockWindow).toHaveBeenCalledWith(qx, PURL, '2026-06-01', '2026-07-01', 300, true) + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, [ + 'downloads_last_30d.count', + 'packages.downloads_last_30d', + ]) + expect(changedFields).toEqual(['downloads_last_30d.count', 'packages.downloads_last_30d']) + }) + + it('returns no changed fields and does not audit when a window for the month already exists', async () => { + mockWindow.mockResolvedValue([]) + + const changedFields = await persistPackagist30dWindow(qx, PURL, 300, '2026-07-15') + + expect(mockWindow).toHaveBeenCalledWith(qx, PURL, '2026-06-01', '2026-07-01', 300, true) + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, []) + expect(changedFields).toEqual([]) + }) + + it('writes nothing and returns no changed fields when the registry reported no monthly count', async () => { + const changedFields = await persistPackagist30dWindow(qx, PURL, null, '2026-07-01') + + expect(mockWindow).not.toHaveBeenCalled() + expect(mockAudit).not.toHaveBeenCalled() + expect(changedFields).toEqual([]) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts new file mode 100644 index 0000000000..af1d616e82 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/dueSelection.test.ts @@ -0,0 +1,291 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getCriticalPackagistPackageCount, + insertPackagistPackages, + updatePackagistPackageStats, +} from '@crowd/data-access-layer/src/packages/packages' +import { + getPackagist30dDuePurls, + getPackagistDailyDownloadsDue, + getPackagistMetadataDuePurls, + markPackagist30dProcessed, + markPackagistDailyProcessed, + markPackagistMetadataScanned, +} from '@crowd/data-access-layer/src/packages/packagistPackageState' +import { upsertPackagistVersions } from '@crowd/data-access-layer/src/packages/versions' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +// Executes the real DAL functions against a capturing stand-in executor — no database. +function makeQx() { + return { + select: vi.fn().mockResolvedValue([]), + selectOne: vi.fn().mockResolvedValue({ count: '0' }), + selectOneOrNone: vi.fn().mockResolvedValue(null), + result: vi.fn().mockResolvedValue(0), + tx: vi.fn(), + } +} +type FakeQx = ReturnType +const asQx = (qx: FakeQx) => qx as unknown as QueryExecutor + +let qx: FakeQx + +beforeEach(() => { + qx = makeQx() +}) + +// Metadata (merged enrichment) due-selection: keyset pagination, refresh window, and +// the stored Last-Modified for If-Modified-Since replay on the p2 fetch. `cutoff` is a +// pre-computed threshold (this run's fixed start time minus the refresh window), not a +// live NOW() — a keyset scan only visits each purl once per drain, so due-selection +// must stay anchored to one stable point in time for the whole run. +describe('getPackagistMetadataDuePurls', () => { + const CUTOFF = '2026-07-08T00:00:00.000Z' + + it('returns purls with their stored Last-Modified, scoped by the fixed cutoff', async () => { + qx.select.mockResolvedValue([ + { purl: 'pkg:composer/a/x', metadata_last_modified: 'Tue, 30 Jun 2026 00:00:00 GMT' }, + { purl: 'pkg:composer/b/y', metadata_last_modified: null }, + ]) + + const candidates = await getPackagistMetadataDuePurls(asQx(qx), CUTOFF, '', 50, true) + + expect(candidates).toEqual([ + { purl: 'pkg:composer/a/x', metadataLastModified: 'Tue, 30 Jun 2026 00:00:00 GMT' }, + { purl: 'pkg:composer/b/y', metadataLastModified: null }, + ]) + const [sql, params] = qx.select.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(sql).toMatch(/is_critical/) + expect(sql).toMatch(/metadata_last_run_at/) + expect(sql).toMatch(/ORDER BY\s+p\.purl/i) + expect(params).toMatchObject({ cutoff: CUTOFF, batchSize: 50, onlyCritical: true }) + }) + + it('selects across all packagist packages when onlyCritical is false (the all-packages default)', async () => { + await getPackagistMetadataDuePurls(asQx(qx), CUTOFF, '', 50, false) + + const [sql, params] = qx.select.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(params).toMatchObject({ onlyCritical: false }) + }) +}) + +describe('markPackagistMetadataScanned', () => { + it('upserts the metadata run result and stores a fresh Last-Modified when given', async () => { + await markPackagistMetadataScanned( + asQx(qx), + 'pkg:composer/a/x', + { status: 'success', attempts: 1 }, + { metadataLastModified: 'Wed, 01 Jul 2026 00:00:00 GMT' }, + ) + + const [sql, params] = qx.result.mock.calls[0] + expect(sql).toMatch(/packagist_package_state/) + expect(sql).toMatch(/metadata_run_result/) + expect(sql).toMatch(/metadata_last_modified/) + expect(sql).toMatch(/ON CONFLICT \(purl\)/) + expect(params).toMatchObject({ + purl: 'pkg:composer/a/x', + result: JSON.stringify({ status: 'success', attempts: 1 }), + metadataLastModified: 'Wed, 01 Jul 2026 00:00:00 GMT', + }) + }) + + it('keeps the stored Last-Modified when none is given', async () => { + await markPackagistMetadataScanned(asQx(qx), 'pkg:composer/a/x', { + status: 'success', + attempts: 1, + }) + + const [sql, params] = qx.result.mock.calls[0] + expect(params).toMatchObject({ metadataLastModified: null }) + expect(sql).toMatch(/COALESCE/i) + }) +}) + +// Monthly downloads-30d lane: npm-style breadth — every package is due once per run +// (watermark older than the run's cutoff, or never run). +describe('getPackagist30dDuePurls / markPackagist30dProcessed', () => { + it('selects packagist purls whose 30d watermark is older than the cutoff, keyset-paginated by purl', async () => { + qx.select.mockResolvedValue([{ purl: 'pkg:composer/a/x' }]) + + const purls = await getPackagist30dDuePurls(asQx(qx), '2026-07-01T03:53:00Z', '', 50) + + expect(purls).toEqual(['pkg:composer/a/x']) + const [sql, params] = qx.select.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(sql).toMatch(/downloads_30d_last_run_at/) + expect(sql).toMatch(/p\.purl > \$\(afterPurl\)/) + expect(sql).toMatch(/ORDER BY\s+p\.purl/i) + expect(sql).toMatch(/LIMIT/i) + expect(params).toMatchObject({ cutoff: '2026-07-01T03:53:00Z', afterPurl: '', batchSize: 50 }) + }) + + it('bumps the 30d watermark with the run result', async () => { + await markPackagist30dProcessed(asQx(qx), 'pkg:composer/a/x', { + status: 'success', + attempts: 1, + }) + + const [sql, params] = qx.result.mock.calls[0] + expect(sql).toMatch(/packagist_package_state/) + expect(sql).toMatch(/downloads_30d_last_run_at/) + expect(sql).toMatch(/downloads_30d_run_result/) + expect(sql).toMatch(/ON CONFLICT \(purl\)/) + expect(params).toMatchObject({ + purl: 'pkg:composer/a/x', + result: JSON.stringify({ status: 'success', attempts: 1 }), + }) + }) +}) + +// Daily downloads lane: critical slice only, cutoff watermark, returns the package id +// the downloads_daily insert needs. +describe('getPackagistDailyDownloadsDue / markPackagistDailyProcessed', () => { + it('selects critical packagist packages due for a daily capture, keyset-paginated by purl', async () => { + qx.select.mockResolvedValue([{ purl: 'pkg:composer/a/x', package_id: '7' }]) + + const due = await getPackagistDailyDownloadsDue(asQx(qx), '2026-07-15T06:23:00Z', '', 50) + + expect(due).toEqual([{ purl: 'pkg:composer/a/x', packageId: '7' }]) + const [sql, params] = qx.select.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(sql).toMatch(/is_critical/) + expect(sql).toMatch(/daily_downloads_last_run_at/) + expect(sql).toMatch(/p\.purl > \$\(afterPurl\)/) + expect(sql).toMatch(/ORDER BY\s+p\.purl/i) + expect(params).toMatchObject({ cutoff: '2026-07-15T06:23:00Z', afterPurl: '', batchSize: 50 }) + }) + + it('bumps the daily watermark with the run result', async () => { + await markPackagistDailyProcessed(asQx(qx), 'pkg:composer/a/x', { + status: 'error', + attempts: 3, + errorKind: 'NOT_FOUND', + }) + + const [sql, params] = qx.result.mock.calls[0] + expect(sql).toMatch(/daily_downloads_last_run_at/) + expect(sql).toMatch(/daily_downloads_run_result/) + expect(sql).toMatch(/ON CONFLICT \(purl\)/) + expect(params).toMatchObject({ purl: 'pkg:composer/a/x' }) + }) +}) + +// Seeding is insert-or-skip: existing rows are never clobbered. +describe('insertPackagistPackages', () => { + it('inserts seed rows with packagist identity and skips existing purls', async () => { + qx.result.mockResolvedValue(2) + + const inserted = await insertPackagistPackages(asQx(qx), [ + { purl: 'pkg:composer/a/x', vendor: 'a', name: 'x' }, + { purl: 'pkg:composer/b/y', vendor: 'b', name: 'y' }, + ]) + + expect(inserted).toBe(2) + const [sql, params] = qx.result.mock.calls[0] + expect(sql).toMatch(/INSERT INTO packages/) + expect(sql).toMatch(/'packagist'/) + expect(sql).toMatch(/'packagist-registry'/) + expect(sql).toMatch(/ON CONFLICT \(purl\) DO NOTHING/) + expect(params).toMatchObject({ + purls: ['pkg:composer/a/x', 'pkg:composer/b/y'], + vendors: ['a', 'b'], + names: ['x', 'y'], + }) + }) + + it('is a no-op for an empty batch', async () => { + expect(await insertPackagistPackages(asQx(qx), [])).toBe(0) + expect(qx.result).not.toHaveBeenCalled() + }) +}) + +// The package-info update targets the packagist row and reports identity for scoping. +describe('updatePackagistPackageStats', () => { + it('returns null when no packagist row matches the purl', async () => { + expect( + await updatePackagistPackageStats(asQx(qx), { + purl: 'pkg:composer/a/x', + description: null, + declaredRepositoryUrl: null, + repositoryUrl: null, + status: 'active', + totalDownloads: 2, + dependentCount: 3, + }), + ).toBeNull() + const [sql] = qx.selectOneOrNone.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + }) + + it('maps the row to id/isCritical/changedFields', async () => { + qx.selectOneOrNone.mockResolvedValue({ + id: '7', + is_critical: true, + changed_fields: ['packages.description'], + }) + + const result = await updatePackagistPackageStats(asQx(qx), { + purl: 'pkg:composer/a/x', + description: 'd', + declaredRepositoryUrl: 'https://github.com/a/x', + repositoryUrl: 'https://github.com/a/x', + status: 'active', + totalDownloads: 2, + dependentCount: 3, + }) + + expect(result).toEqual({ + id: '7', + isCritical: true, + changedFields: ['packages.description'], + }) + }) +}) + +// The guard input: how many critical packagist packages exist. +describe('getCriticalPackagistPackageCount', () => { + it('counts critical packagist packages', async () => { + qx.selectOne.mockResolvedValue({ count: '12' }) + + expect(await getCriticalPackagistPackageCount(asQx(qx))).toBe(12) + const [sql] = qx.selectOne.mock.calls[0] + expect(sql).toMatch(/ecosystem = 'packagist'/) + expect(sql).toMatch(/is_critical/) + }) +}) + +// The stale-is_latest cleanup only ever flips rows OUTSIDE the upsert batch (batch rows +// already had is_latest set by the upsert CTE), so its row count must feed changedFields +// separately or those real changes never reach the audit log. +describe('upsertPackagistVersions is_latest cleanup audit', () => { + const versions = [ + { number: '2.0.0', publishedAt: null, isLatest: true, isPrerelease: false, licenses: null }, + ] + + it("appends versions.is_latest when the cleanup cleared a stale row the CTE diff can't see", async () => { + qx.selectOne.mockResolvedValue({ changed_fields: [], version_ids: [] }) + qx.result.mockResolvedValue(1) + + const { changedFields } = await upsertPackagistVersions(asQx(qx), '7', versions, '2.0.0') + + const [cleanupSql] = qx.result.mock.calls[0] + expect(cleanupSql).toMatch(/is_latest = false/) + expect(changedFields).toContain('versions.is_latest') + }) + + it('reports nothing extra when the cleanup cleared no rows, and never duplicates the CTE diff', async () => { + qx.selectOne.mockResolvedValue({ changed_fields: ['versions.is_latest'], version_ids: [] }) + qx.result.mockResolvedValue(0) + + const noClear = await upsertPackagistVersions(asQx(qx), '7', versions, '2.0.0') + expect(noClear.changedFields).toEqual(['versions.is_latest']) + + qx.result.mockResolvedValue(2) + const cleared = await upsertPackagistVersions(asQx(qx), '7', versions, '2.0.0') + expect(cleared.changedFields.filter((f) => f === 'versions.is_latest')).toHaveLength(1) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/expandMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/expandMetadata.test.ts new file mode 100644 index 0000000000..b257d41f54 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/expandMetadata.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest' + +import { expandComposerMetadata } from '../expandMetadata' + +// C1 — Composer\MetadataMinifier::expand semantics: the first version object is complete, +// each later object carries only changed keys, '__unset' removes a key. +describe('expandComposerMetadata', () => { + it('expands diffs against the previous version and honors __unset', () => { + const expanded = expandComposerMetadata([ + { + version: '3.0.0', + version_normalized: '3.0.0.0', + time: '2026-01-01T00:00:00+00:00', + license: ['MIT'], + require: { php: '>=8.1', 'psr/log': '^3.0' }, + homepage: 'https://example.org', + }, + { + version: '2.0.0', + version_normalized: '2.0.0.0', + time: '2024-01-01T00:00:00+00:00', + require: { php: '>=7.4', 'psr/log': '^2.0' }, + homepage: '__unset', + }, + { + version: '1.0.0', + version_normalized: '1.0.0.0', + time: '2020-01-01T00:00:00+00:00', + license: ['BSD-3-Clause'], + }, + ]) + + expect(expanded).toEqual([ + { + version: '3.0.0', + version_normalized: '3.0.0.0', + time: '2026-01-01T00:00:00+00:00', + license: ['MIT'], + require: { php: '>=8.1', 'psr/log': '^3.0' }, + homepage: 'https://example.org', + }, + { + // license carried over from 3.0.0, homepage removed via __unset + version: '2.0.0', + version_normalized: '2.0.0.0', + time: '2024-01-01T00:00:00+00:00', + license: ['MIT'], + require: { php: '>=7.4', 'psr/log': '^2.0' }, + }, + { + // require carried over from 2.0.0, license overridden + version: '1.0.0', + version_normalized: '1.0.0.0', + time: '2020-01-01T00:00:00+00:00', + license: ['BSD-3-Clause'], + require: { php: '>=7.4', 'psr/log': '^2.0' }, + }, + ]) + }) + + it('returns independent objects (mutating one expanded version does not leak into others)', () => { + const expanded = expandComposerMetadata([ + { version: '2.0.0', require: { php: '>=8.0' } }, + { version: '1.0.0' }, + ]) + ;(expanded[0] as Record).extra = 'mutated' + expect('extra' in expanded[1]).toBe(false) + }) + + it('returns [] for an empty version list', () => { + expect(expandComposerMetadata([])).toEqual([]) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts new file mode 100644 index 0000000000..ac6cc3fde4 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/fetchPackage.test.ts @@ -0,0 +1,252 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { buildPackagistUserAgent, fetchPackagistP2, fetchPackagistStats } from '../fetchPackage' + +// Minimal Response stand-in for the global fetch mock. `body.cancel` is a spy so +// error-path tests can assert the response body is drained before returning. +function fakeResponse( + status: number, + body?: unknown, + opts: { jsonThrows?: boolean; lastModified?: string } = {}, +): Response & { body: { cancel: ReturnType } } { + return { + status, + ok: status >= 200 && status < 300, + headers: { + get: (k: string) => + k.toLowerCase() === 'last-modified' ? (opts.lastModified ?? null) : null, + }, + json: async () => { + if (opts.jsonThrows) throw new Error('bad json') + return body + }, + body: { cancel: vi.fn() }, + } as unknown as Response & { body: { cancel: ReturnType } } +} + +const validStats = { package: { name: 'monolog/monolog', downloads: { monthly: 5 } } } +const validP2 = { + packages: { 'monolog/monolog': [{ version: '2.0.0' }, { version: '1.0.0' }] }, + minified: 'composer/2.0', +} + +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() + delete process.env.CROWD_PACKAGES_PACKAGIST_MAILTO +}) + +// B1 — polite crawling: UA carries a mailto, configurable via env. +describe('buildPackagistUserAgent', () => { + it('includes a mailto contact by default', () => { + expect(buildPackagistUserAgent()).toMatch(/mailto=/) + }) + + it('uses the env-configured mailto when set', () => { + process.env.CROWD_PACKAGES_PACKAGIST_MAILTO = 'oss@example.org' + expect(buildPackagistUserAgent()).toContain('mailto=oss@example.org') + }) +}) + +// B1 — dynamic (stats) endpoint: status → FetchError kind mapping +describe('fetchPackagistStats', () => { + it('returns the parsed body on 200 with a valid shape, from the dynamic endpoint URL', async () => { + const fetchMock = vi.fn().mockResolvedValue(fakeResponse(200, validStats)) + vi.stubGlobal('fetch', fetchMock) + + expect(await fetchPackagistStats('monolog/monolog')).toEqual(validStats) + expect(fetchMock.mock.calls[0][0]).toBe('https://packagist.org/packages/monolog/monolog.json') + }) + + it('sends a User-Agent containing a mailto', async () => { + const fetchMock = vi.fn().mockResolvedValue(fakeResponse(200, validStats)) + vi.stubGlobal('fetch', fetchMock) + + await fetchPackagistStats('monolog/monolog') + const headers = fetchMock.mock.calls[0][1].headers as Record + expect(headers['User-Agent']).toMatch(/mailto=/) + }) + + it('maps 404 → NOT_FOUND and drains the response body', async () => { + const res = fakeResponse(404) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(res)) + expect(await fetchPackagistStats('gone/gone')).toMatchObject({ + kind: 'NOT_FOUND', + statusCode: 404, + }) + // an undrained body can pin the socket instead of returning it to the shared pool + expect(res.body.cancel).toHaveBeenCalled() + }) + + it('maps 429 → RATE_LIMIT and drains the response body', async () => { + const res = fakeResponse(429) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(res)) + expect(await fetchPackagistStats('busy/busy')).toMatchObject({ + kind: 'RATE_LIMIT', + statusCode: 429, + }) + expect(res.body.cancel).toHaveBeenCalled() + }) + + it('maps other non-ok statuses (5xx) → TRANSIENT with the status code and drains the body', async () => { + const res = fakeResponse(503) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(res)) + expect(await fetchPackagistStats('flaky/flaky')).toMatchObject({ + kind: 'TRANSIENT', + statusCode: 503, + }) + expect(res.body.cancel).toHaveBeenCalled() + }) + + it('maps a network rejection → TRANSIENT without a status code', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('ECONNRESET'))) + const result = await fetchPackagistStats('monolog/monolog') + expect(result).toMatchObject({ kind: 'TRANSIENT' }) + expect((result as { statusCode?: number }).statusCode).toBeUndefined() + }) + + it('maps an unparseable body → MALFORMED', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(fakeResponse(200, undefined, { jsonThrows: true })), + ) + expect(await fetchPackagistStats('monolog/monolog')).toMatchObject({ kind: 'MALFORMED' }) + }) + + it('maps an unexpected JSON shape → MALFORMED', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(200, { not: 'a package' }))) + expect(await fetchPackagistStats('monolog/monolog')).toMatchObject({ kind: 'MALFORMED' }) + }) + + // Regression: a technically-valid-JSON body with the wrong runtime type for a field + // normalizePackagistStats consumes unconditionally must be classified MALFORMED, not + // thrown past the guard (where it would be misread as a transient failure and + // retried forever on the same deterministic input). + it.each([ + ['description is a number', { package: { name: 'a/b', description: 123 } }], + ['repository is an object', { package: { name: 'a/b', repository: {} } }], + ['maintainers is not an array', { package: { name: 'a/b', maintainers: {} } }], + ])('maps a wrong-typed field (%s) → MALFORMED, not a throw', async (_desc, body) => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(200, body))) + await expect(fetchPackagistStats('monolog/monolog')).resolves.toMatchObject({ + kind: 'MALFORMED', + }) + }) + + it('maps a body read aborted by the 30s timeout → TRANSIENT (not MALFORMED)', async () => { + vi.useFakeTimers() + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + status: 200, + ok: true, + headers: { get: () => null }, + json: () => + new Promise((_resolve, reject) => setTimeout(() => reject(new Error('aborted')), 40_000)), + } as unknown as Response), + ) + const p = fetchPackagistStats('monolog/monolog') + await vi.advanceTimersByTimeAsync(41_000) + expect(await p).toMatchObject({ kind: 'TRANSIENT' }) + }) +}) + +// B1 + C5 — static p2 endpoint: same error contract plus If-Modified-Since / 304 handling. +describe('fetchPackagistP2', () => { + it('returns the minified versions and Last-Modified on 200, from the static endpoint URL', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue( + fakeResponse(200, validP2, { lastModified: 'Wed, 01 Jul 2026 00:00:00 GMT' }), + ) + vi.stubGlobal('fetch', fetchMock) + + const result = await fetchPackagistP2('monolog/monolog', null) + expect(result).toEqual({ + minifiedVersions: [{ version: '2.0.0' }, { version: '1.0.0' }], + lastModified: 'Wed, 01 Jul 2026 00:00:00 GMT', + }) + expect(fetchMock.mock.calls[0][0]).toBe('https://repo.packagist.org/p2/monolog/monolog.json') + }) + + it('sends If-Modified-Since when a previous Last-Modified is known', async () => { + const fetchMock = vi.fn().mockResolvedValue(fakeResponse(200, validP2)) + vi.stubGlobal('fetch', fetchMock) + + await fetchPackagistP2('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT') + const headers = fetchMock.mock.calls[0][1].headers as Record + expect(headers['If-Modified-Since']).toBe('Tue, 30 Jun 2026 00:00:00 GMT') + }) + + it('omits If-Modified-Since when none is known', async () => { + const fetchMock = vi.fn().mockResolvedValue(fakeResponse(200, validP2)) + vi.stubGlobal('fetch', fetchMock) + + await fetchPackagistP2('monolog/monolog', null) + const headers = fetchMock.mock.calls[0][1].headers as Record + expect('If-Modified-Since' in headers).toBe(false) + }) + + it('maps 304 → NOT_MODIFIED', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(fakeResponse(304))) + expect(await fetchPackagistP2('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT')).toEqual({ + kind: 'NOT_MODIFIED', + }) + }) + + it('maps 404 → NOT_FOUND and 429 → RATE_LIMIT, draining the body each time', async () => { + const notFound = fakeResponse(404) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(notFound)) + expect(await fetchPackagistP2('gone/gone', null)).toMatchObject({ kind: 'NOT_FOUND' }) + expect(notFound.body.cancel).toHaveBeenCalled() + + const rateLimited = fakeResponse(429) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(rateLimited)) + expect(await fetchPackagistP2('busy/busy', null)).toMatchObject({ kind: 'RATE_LIMIT' }) + expect(rateLimited.body.cancel).toHaveBeenCalled() + }) + + it('maps a payload missing the package key → MALFORMED', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(fakeResponse(200, { packages: { 'other/pkg': [] } })), + ) + expect(await fetchPackagistP2('monolog/monolog', null)).toMatchObject({ kind: 'MALFORMED' }) + }) + + // Regression: a malformed element in the version array must be classified + // MALFORMED, not thrown past the guard — expandComposerMetadata's Object.entries() + // throws on a null/non-object element, and version/version_normalized/license reach + // .startsWith()/.split()/.endsWith() and the SQL text[] write unconditionally. + it.each([ + ['an element is null', [null]], + ['version is missing', [{ time: '2024-01-01' }]], + ['version is a number', [{ version: 123 }]], + ['version_normalized is a number', [{ version: '1.0.0', version_normalized: 42 }]], + ['license is a scalar string, not an array', [{ version: '1.0.0', license: 'MIT' }]], + ['homepage is an object', [{ version: '1.0.0', homepage: {} }]], + ['time is a number', [{ version: '1.0.0', time: 20240101 }]], + ['time is a string but not a parseable date', [{ version: '1.0.0', time: 'not-a-date' }]], + ])('maps a malformed version entry (%s) → MALFORMED, not a throw', async (_desc, versions) => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(fakeResponse(200, { packages: { 'monolog/monolog': versions } })), + ) + await expect(fetchPackagistP2('monolog/monolog', null)).resolves.toMatchObject({ + kind: 'MALFORMED', + }) + }) + + it('accepts a version entry using the __unset diff sentinel on optional fields', async () => { + const versions = [ + { version: '1.0.0', license: ['MIT'] }, + { version: '2.0.0', license: '__unset' }, + ] + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(fakeResponse(200, { packages: { 'monolog/monolog': versions } })), + ) + const result = await fetchPackagistP2('monolog/monolog', null) + expect(result).not.toMatchObject({ kind: 'MALFORMED' }) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts new file mode 100644 index 0000000000..cb79a500b5 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/ingest.test.ts @@ -0,0 +1,501 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getPackagistMetadataDuePurls, + insertDailyDownloads, + insertPackagistPackages, + logAuditFieldChanges, + markPackagist30dProcessed, + markPackagistDailyProcessed, + markPackagistMetadataScanned, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { + getPackagistMetadataBatch, + ingestOnePackagist30dWindow, + ingestOnePackagistDailyDownload, + ingestOnePackagistMetadata, + ingestPackagistItemsConcurrently, + runPackagistPackageSeed, +} from '../activities' +import { persistPackagist30dWindow } from '../downloads' +import { expandComposerMetadata } from '../expandMetadata' +import { fetchPackagistP2, fetchPackagistStats } from '../fetchPackage' +import { fetchPackagistPackageList, parsePackagistPackageList } from '../listPackages' +import { INGEST_MAX_ATTEMPTS } from '../retryPolicy' +import { persistPackagistMetadata } from '../upsertMetadata' +import { persistPackagistPackageInfo } from '../upsertPackageInfo' + +// Mock the heavy collaborators so we exercise the classification/retry/give-up logic. +vi.mock('../fetchPackage', () => ({ + fetchPackagistStats: vi.fn(), + fetchPackagistP2: vi.fn(), + buildPackagistUserAgent: vi.fn(() => 'ua'), +})) +vi.mock('../upsertPackageInfo', () => ({ persistPackagistPackageInfo: vi.fn() })) +vi.mock('../upsertMetadata', () => ({ persistPackagistMetadata: vi.fn() })) +vi.mock('../downloads', () => ({ + monthlyWindowFor: vi.fn(), + persistPackagist30dWindow: vi.fn(), +})) +vi.mock('../expandMetadata', () => ({ expandComposerMetadata: vi.fn() })) +vi.mock('../listPackages', () => ({ + fetchPackagistPackageList: vi.fn(), + parsePackagistPackageList: vi.fn(), +})) +vi.mock('@crowd/data-access-layer/src/packages', () => ({ + markPackagistMetadataScanned: vi.fn(), + markPackagist30dProcessed: vi.fn(), + markPackagistDailyProcessed: vi.fn(), + getPackagistMetadataDuePurls: vi.fn(), + getPackagist30dDuePurls: vi.fn(), + getPackagistDailyDownloadsDue: vi.fn(), + getCriticalPackagistPackageCount: vi.fn(), + insertDailyDownloads: vi.fn().mockResolvedValue([]), + insertPackagistPackages: vi.fn(), + logAuditFieldChanges: vi.fn(), +})) + +const mockFetchStats = vi.mocked(fetchPackagistStats) +const mockFetchP2 = vi.mocked(fetchPackagistP2) +const mockExpand = vi.mocked(expandComposerMetadata) +const mockPersistInfo = vi.mocked(persistPackagistPackageInfo) +const mockPersistMetadata = vi.mocked(persistPackagistMetadata) +const mockPersist30d = vi.mocked(persistPackagist30dWindow) +const mockDaily = vi.mocked(insertDailyDownloads) +const mockMarkMetadata = vi.mocked(markPackagistMetadataScanned) +const mockMark30d = vi.mocked(markPackagist30dProcessed) +const mockMarkDaily = vi.mocked(markPackagistDailyProcessed) +const mockAudit = vi.mocked(logAuditFieldChanges) +const mockFetchList = vi.mocked(fetchPackagistPackageList) +const mockParseList = vi.mocked(parsePackagistPackageList) +const mockSeedInsert = vi.mocked(insertPackagistPackages) + +const qx = { + tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), +} as unknown as QueryExecutor +const PURL = 'pkg:composer/monolog/monolog' +const RUN_DATE = '2026-07-15' +const SCHEDULED_AT = '2026-07-15T00:00:00.000Z' + +const statsJson = { + package: { name: 'monolog/monolog', downloads: { total: 1000, monthly: 300, daily: 10 } }, +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +afterEach(() => { + vi.useRealTimers() +}) + +// The merged metadata lane: one pass per package fetches BOTH registry endpoints — +// the dynamic one (package info, repos, maintainers) and p2 (versions, dependencies). +describe('ingestOnePackagistMetadata', () => { + const candidate = { purl: PURL, metadataLastModified: 'Tue, 30 Jun 2026 00:00:00 GMT' } + const minified = [{ version: '2.0.0' }] + const expanded = [{ version: '2.0.0', version_normalized: '2.0.0.0' }] + + function happyMocks() { + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) + mockFetchP2.mockResolvedValue({ + minifiedVersions: minified, + lastModified: 'Wed, 01 Jul 2026 00:00:00 GMT', + } as never) + mockExpand.mockReturnValue(expanded as never) + mockPersistMetadata.mockResolvedValue({ + found: true, + changedFields: ['versions.number'], + unresolvedDependencyTargets: 0, + }) + } + + it('persists phase 1 immediately and phase 2 separately, then marks scanned with the fresh Last-Modified', async () => { + happyMocks() + + await ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) + + expect(mockPersistInfo).toHaveBeenCalledWith(qx, PURL, expect.anything()) + expect(mockFetchP2).toHaveBeenCalledWith('monolog/monolog', 'Tue, 30 Jun 2026 00:00:00 GMT') + expect(mockExpand).toHaveBeenCalledWith(minified) + expect(mockPersistMetadata).toHaveBeenCalledWith(qx, PURL, expanded) + // Each phase audits its own writes atomically inside its own persist* call (see + // persistPackageInfo.test.ts / persistMetadata.test.ts) — phase 1 is committed and + // audited before the p2 fetch (which can throw) ever runs. + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success', attempts: 1 }), + { metadataLastModified: 'Wed, 01 Jul 2026 00:00:00 GMT' }, + ) + }) + + it('records a p2 NOT_MODIFIED as success: info persisted, versions skipped', async () => { + happyMocks() + mockFetchP2.mockResolvedValue({ kind: 'NOT_MODIFIED' } as never) + + await ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) + + expect(mockPersistInfo).toHaveBeenCalled() + expect(mockPersistMetadata).not.toHaveBeenCalled() + // no fresh Last-Modified to replay when p2 wasn't refetched + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success' }), + { metadataLastModified: null }, + ) + }) + + it('gives up on a persistent dynamic 404 without touching p2', async () => { + vi.useFakeTimers() + mockFetchStats.mockResolvedValue({ + kind: 'NOT_FOUND', + statusCode: 404, + message: 'not found', + } as never) + + const p = ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) + await vi.runAllTimersAsync() + await p + + expect(mockFetchStats).toHaveBeenCalledTimes(3) + expect(mockFetchP2).not.toHaveBeenCalled() + expect(mockPersistInfo).not.toHaveBeenCalled() + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND', httpStatus: 404 }), + { notBefore: SCHEDULED_AT }, + ) + }) + + it('throws on a transient dynamic result without marking scanned', async () => { + mockFetchStats.mockResolvedValue({ + kind: 'TRANSIENT', + statusCode: 500, + message: 'HTTP 500', + } as never) + + await expect(ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT)).rejects.toThrow() + expect(mockMarkMetadata).not.toHaveBeenCalled() + expect(mockFetchStats).toHaveBeenCalledTimes(1) + }) + + it('gives up on a persistent p2 404 after fast retries and marks it scanned(error)', async () => { + vi.useFakeTimers() + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: [] }) + mockFetchP2.mockResolvedValue({ + kind: 'NOT_FOUND', + statusCode: 404, + message: 'not found', + } as never) + + const p = ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) + await vi.runAllTimersAsync() + await p + + expect(mockPersistMetadata).not.toHaveBeenCalled() + // p2-only failure must not bump metadata_last_run_at — phase 1 already succeeded, + // but versions/deps never refreshed, so the package must stay due + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + { bumpLastRunAt: false, notBefore: SCHEDULED_AT }, + ) + }) + + it('still persists (and self-audits) phase 1 even when the p2 fetch gives up', async () => { + // the dynamic-endpoint writes are committed and audited by persistPackagistPackageInfo + // itself before the p2 fetch (which can throw) ever runs — see persistPackageInfo.test.ts + vi.useFakeTimers() + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) + mockFetchP2.mockResolvedValue({ + kind: 'NOT_FOUND', + statusCode: 404, + message: 'not found', + } as never) + + const p = ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT) + await vi.runAllTimersAsync() + await p + + expect(mockPersistInfo).toHaveBeenCalledWith(qx, PURL, expect.anything()) + expect(mockMarkMetadata).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + { bumpLastRunAt: false, notBefore: SCHEDULED_AT }, + ) + }) + + it('throws on a transient p2 result without marking scanned, but phase 1 already persisted', async () => { + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersistInfo.mockResolvedValue({ found: true, changedFields: ['packages.description'] }) + mockFetchP2.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 502' } as never) + + await expect(ingestOnePackagistMetadata(qx, candidate, SCHEDULED_AT)).rejects.toThrow() + expect(mockMarkMetadata).not.toHaveBeenCalled() + // phase-1 writes (and their audit row) are already committed by + // persistPackagistPackageInfo when the throw happens — a retry re-runs phase 1 + // idempotently and reports no changes, so nothing here can lose that audit event + expect(mockPersistInfo).toHaveBeenCalledWith(qx, PURL, expect.anything()) + }) +}) + +// The monthly downloads-30d lane: dynamic fetch, window row only. +describe('ingestOnePackagist30dWindow', () => { + it('persists the observed rolling window (self-audited) and marks the run processed', async () => { + mockFetchStats.mockResolvedValue(statsJson as never) + mockPersist30d.mockResolvedValue(['downloads_last_30d.count', 'packages.downloads_last_30d']) + + await ingestOnePackagist30dWindow(qx, PURL, RUN_DATE, SCHEDULED_AT) + + expect(mockPersist30d).toHaveBeenCalledWith(qx, PURL, 300, RUN_DATE) + // persistPackagist30dWindow audits its own write atomically — see downloads.test.ts + expect(mockMark30d).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success', attempts: 1 }), + ) + }) + + it('gives up on a persistent 404 and marks the run errored', async () => { + vi.useFakeTimers() + mockFetchStats.mockResolvedValue({ + kind: 'NOT_FOUND', + statusCode: 404, + message: 'not found', + } as never) + + const p = ingestOnePackagist30dWindow(qx, PURL, RUN_DATE, SCHEDULED_AT) + await vi.runAllTimersAsync() + await p + + expect(mockPersist30d).not.toHaveBeenCalled() + expect(mockMark30d).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'error', errorKind: 'NOT_FOUND' }), + SCHEDULED_AT, + ) + }) + + it('throws on a transient result without marking processed', async () => { + mockFetchStats.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 503' } as never) + + await expect(ingestOnePackagist30dWindow(qx, PURL, RUN_DATE, SCHEDULED_AT)).rejects.toThrow() + expect(mockMark30d).not.toHaveBeenCalled() + }) +}) + +// The daily downloads lane (critical slice): dynamic fetch, one downloads_daily row. +describe('ingestOnePackagistDailyDownload', () => { + const candidate = { purl: PURL, packageId: '7' } + + it('inserts the daily row, audits the change, and marks the run processed', async () => { + mockFetchStats.mockResolvedValue(statsJson as never) + mockDaily.mockResolvedValue(['downloads_daily.date', 'downloads_daily.count']) + + await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE, SCHEDULED_AT) + + expect(mockDaily).toHaveBeenCalledWith(qx, '7', [{ day: RUN_DATE, downloads: 10 }]) + expect(mockAudit).toHaveBeenCalledWith(qx, 'packagist', PURL, [ + 'downloads_daily.date', + 'downloads_daily.count', + ]) + expect(mockMarkDaily).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success' }), + ) + }) + + it('marks success without inserting or auditing when the registry reports no daily count', async () => { + mockFetchStats.mockResolvedValue({ package: { name: 'monolog/monolog' } } as never) + + await ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE, SCHEDULED_AT) + + expect(mockDaily).not.toHaveBeenCalled() + expect(mockAudit).not.toHaveBeenCalled() + expect(mockMarkDaily).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ status: 'success' }), + ) + }) + + it('throws on a transient result without marking processed', async () => { + mockFetchStats.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 503' } as never) + + await expect( + ingestOnePackagistDailyDownload(qx, candidate, RUN_DATE, SCHEDULED_AT), + ).rejects.toThrow() + expect(mockMarkDaily).not.toHaveBeenCalled() + }) +}) + +// Concurrency-bounded batch processing with the pypi give-up contract: transient failures +// rethrow while Temporal retries remain, then are given up per-item so the cursor advances. +describe('ingestPackagistItemsConcurrently', () => { + const items = ['a', 'bad', 'c', 'd', 'e', 'f'] + const makeIngest = () => + vi.fn((item: string) => + item === 'bad' ? Promise.reject(new Error('transient')) : Promise.resolve(), + ) + + it('rethrows while Temporal retries remain and does not give up on any item', async () => { + const ingest = makeIngest() + const onGiveUp = vi.fn().mockResolvedValue(undefined) + + await expect(ingestPackagistItemsConcurrently(items, 1, 2, ingest, onGiveUp)).rejects.toThrow( + 'transient', + ) + expect(onGiveUp).not.toHaveBeenCalled() + // every item — including ones scheduled after 'bad' — must get a genuine attempt + // this round; `attempt` tracks the whole batch, not each item, so a rethrow from + // inside the concurrency pool must never starve later items of their own try. + expect(ingest).toHaveBeenCalledTimes(6) + }) + + it('still attempts every later item when the FIRST item in the batch fails (sequential, deterministic)', async () => { + const ingest = vi.fn((item: string) => + item === 'bad' ? Promise.reject(new Error('transient')) : Promise.resolve(), + ) + const onGiveUp = vi.fn().mockResolvedValue(undefined) + + // concurrency=1 forces full sequencing: mapWithConcurrency awaits each item's + // settlement (including the old code's rethrow-triggered `failed=true`) before + // ever considering the next index, so this deterministically proves items after + // the failing one still get scheduled instead of being starved. + await expect( + ingestPackagistItemsConcurrently(['bad', 'ok1', 'ok2'], 1, 1, ingest, onGiveUp), + ).rejects.toThrow('transient') + + expect(ingest).toHaveBeenCalledTimes(3) + expect(onGiveUp).not.toHaveBeenCalled() + }) + + it('on the final attempt gives up on the failing item and completes the rest', async () => { + const ingest = makeIngest() + const onGiveUp = vi.fn().mockResolvedValue(undefined) + + await expect( + ingestPackagistItemsConcurrently(items, INGEST_MAX_ATTEMPTS, 2, ingest, onGiveUp), + ).resolves.toBeUndefined() + + expect(ingest).toHaveBeenCalledTimes(6) + expect(onGiveUp).toHaveBeenCalledTimes(1) + expect(onGiveUp.mock.calls[0][0]).toBe('bad') + }) + + it('never runs more than `concurrency` ingests at once', async () => { + let inFlight = 0 + let maxInFlight = 0 + const ingest = vi.fn(async () => { + inFlight += 1 + maxInFlight = Math.max(maxInFlight, inFlight) + await new Promise((resolve) => setTimeout(resolve, 5)) + inFlight -= 1 + }) + + await ingestPackagistItemsConcurrently(items, 1, 2, ingest, vi.fn()) + + expect(ingest).toHaveBeenCalledTimes(6) + expect(maxInFlight).toBeLessThanOrEqual(2) + }) +}) + +// All-packages metadata scope: the env default drives due-selection's onlyCritical arg. +// Default is ALL packages (deps.dev has no Packagist data to fall back on); the env var +// narrows back to the critical slice. The due-cutoff is derived once from the run's +// fixed cutoff minus the refresh window, not a live NOW() re-evaluated per batch. +describe('getPackagistMetadataBatch scope', () => { + const mockMetadataDue = vi.mocked(getPackagistMetadataDuePurls) + const RUN_CUTOFF = '2026-07-15T00:00:00.000Z' + const DUE_CUTOFF = '2026-07-08T00:00:00.000Z' // RUN_CUTOFF minus the 7-day default + + afterEach(() => { + delete process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL + delete process.env.CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS + }) + + it('derives the due-cutoff from the fixed run cutoff, not a live clock', async () => { + process.env.CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS = '3' + mockMetadataDue.mockResolvedValue([]) + + await getPackagistMetadataBatch(RUN_CUTOFF, '', 50) + + // 3 days before the run's own fixed cutoff — never "3 days before whenever this + // particular batch happens to execute" — is what makes the keyset scan give every + // purl exactly one refresh-eligibility check per drain. + expect(mockMetadataDue).toHaveBeenCalledWith( + expect.anything(), + '2026-07-12T00:00:00.000Z', + '', + 50, + false, + ) + }) + + it('selects across ALL packagist packages by default', async () => { + mockMetadataDue.mockResolvedValue([]) + + await getPackagistMetadataBatch(RUN_CUTOFF, '', 50) + + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), DUE_CUTOFF, '', 50, false) + }) + + it('narrows to the critical slice when CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL=true', async () => { + process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL = 'true' + mockMetadataDue.mockResolvedValue([]) + + await getPackagistMetadataBatch(RUN_CUTOFF, '', 50) + + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), DUE_CUTOFF, '', 50, true) + }) + + it('keeps the all-packages default when the env value is unrecognized', async () => { + // a typo must not silently flip the sweep to critical-only + process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL = 'ture' + mockMetadataDue.mockResolvedValue([]) + + await getPackagistMetadataBatch(RUN_CUTOFF, '', 50) + + expect(mockMetadataDue).toHaveBeenCalledWith(expect.anything(), DUE_CUTOFF, '', 50, false) + }) +}) + +// A1/A2 — the seed activity: one list fetch, parsed entries inserted, counts reported. +describe('runPackagistPackageSeed', () => { + it('fetches the list once, seeds all parsed entries, and reports counts', async () => { + const entries = [ + { vendor: 'a', name: 'x', purl: 'pkg:composer/a/x' }, + { vendor: 'b', name: 'y', purl: 'pkg:composer/b/y' }, + { vendor: 'c', name: 'z', purl: 'pkg:composer/c/z' }, + ] + mockFetchList.mockResolvedValue({ packageNames: ['raw'] }) + mockParseList.mockReturnValue({ entries, invalid: 2 }) + mockSeedInsert.mockResolvedValue(3) + + const result = await runPackagistPackageSeed() + + expect(mockFetchList).toHaveBeenCalledTimes(1) + const seeded = mockSeedInsert.mock.calls.flatMap((c) => c[1]) + expect(seeded).toEqual(entries) + expect(result).toEqual({ discovered: 3, invalid: 2 }) + }) + + it('throws when the list fetch fails so Temporal retries the seed', async () => { + mockFetchList.mockResolvedValue({ kind: 'TRANSIENT', message: 'HTTP 502' }) + + await expect(runPackagistPackageSeed()).rejects.toThrow() + expect(mockSeedInsert).not.toHaveBeenCalled() + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts new file mode 100644 index 0000000000..76bfb68d62 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/listPackages.test.ts @@ -0,0 +1,139 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { fetchPackagistPackageList, parsePackagistPackageList } from '../listPackages' + +afterEach(() => { + vi.unstubAllGlobals() +}) + +// A1 — list.json parsing: vendor/name split, purl form, invalid skipped+counted, dedup. +describe('parsePackagistPackageList', () => { + it('maps names to vendor/name entries with pkg:composer purls', () => { + const { entries, invalid } = parsePackagistPackageList({ + packageNames: ['monolog/monolog', 'symfony/http-kernel'], + }) + expect(entries).toEqual([ + { vendor: 'monolog', name: 'monolog', purl: 'pkg:composer/monolog/monolog' }, + { vendor: 'symfony', name: 'http-kernel', purl: 'pkg:composer/symfony/http-kernel' }, + ]) + expect(invalid).toBe(0) + }) + + it('accepts dots, underscores and hyphens in segments', () => { + const { entries } = parsePackagistPackageList({ + packageNames: ['foo/bar.baz', 'a1/b_c', 'x-y/z-1'], + }) + expect(entries.map((e) => e.purl)).toEqual([ + 'pkg:composer/foo/bar.baz', + 'pkg:composer/a1/b_c', + 'pkg:composer/x-y/z-1', + ]) + }) + + it('accepts up to two consecutive hyphens in the project segment only', () => { + // Composer's own name spec allows `-{1,2}` as a separator in the project segment + // but only a single separator in the vendor segment. + const { entries, invalid } = parsePackagistPackageList({ + packageNames: ['vendor/my--package', 'vendor/my---package', 'my--vendor/package'], + }) + expect(entries.map((e) => e.purl)).toEqual(['pkg:composer/vendor/my--package']) + expect(invalid).toBe(2) + }) + + it('skips and counts invalid names', () => { + const { entries, invalid } = parsePackagistPackageList({ + packageNames: [ + 'valid/name', + 'noslash', + 'too/many/parts', + '/leading', + 'trailing/', + 'has space/pkg', + '', + 42, + ], + }) + expect(entries.map((e) => e.purl)).toEqual(['pkg:composer/valid/name']) + expect(invalid).toBe(7) + }) + + it('rejects a long invalid segment without catastrophic regex backtracking', () => { + // CodeQL js/redos: a long run of one char class followed by a char that breaks + // the match is the classic trigger for exponential backtracking. + const pathological = 'vendor/' + '0'.repeat(50) + '!' + const start = Date.now() + const { entries, invalid } = parsePackagistPackageList({ packageNames: [pathological] }) + expect(Date.now() - start).toBeLessThan(200) + expect(entries).toEqual([]) + expect(invalid).toBe(1) + }) + + it('rejects a long run of hyphens without catastrophic backtracking (project -{1,2} alternative)', () => { + const pathological = 'a' + '-'.repeat(50) + '!/name' + const start = Date.now() + const { entries, invalid } = parsePackagistPackageList({ packageNames: [pathological] }) + expect(Date.now() - start).toBeLessThan(200) + expect(entries).toEqual([]) + expect(invalid).toBe(1) + }) + + it('lowercases and deduplicates names without counting duplicates as invalid', () => { + const { entries, invalid } = parsePackagistPackageList({ + packageNames: ['Monolog/Monolog', 'monolog/monolog'], + }) + expect(entries).toEqual([ + { vendor: 'monolog', name: 'monolog', purl: 'pkg:composer/monolog/monolog' }, + ]) + expect(invalid).toBe(0) + }) + + it('throws on a root shape without a packageNames array', () => { + expect(() => parsePackagistPackageList({ nope: true })).toThrow() + expect(() => parsePackagistPackageList({ packageNames: 'not-an-array' })).toThrow() + expect(() => parsePackagistPackageList(null)).toThrow() + }) +}) + +describe('fetchPackagistPackageList', () => { + it('returns the raw JSON on 200 and hits the list endpoint', async () => { + const body = { packageNames: ['monolog/monolog'] } + const fetchMock = vi.fn().mockResolvedValue({ + status: 200, + ok: true, + json: async () => body, + } as unknown as Response) + vi.stubGlobal('fetch', fetchMock) + + expect(await fetchPackagistPackageList()).toEqual(body) + expect(fetchMock.mock.calls[0][0]).toBe('https://packagist.org/packages/list.json') + }) + + it('maps a 5xx to TRANSIENT', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ status: 503, ok: false, json: async () => ({}) }), + ) + expect(await fetchPackagistPackageList()).toMatchObject({ kind: 'TRANSIENT', statusCode: 503 }) + }) + + // Regression: the seed's own error paths share the same 10-connection dispatcher as + // the stats/p2 fetches, so an undrained body here risks the same socket exhaustion. + it.each([ + ['404', 404, 'NOT_FOUND'], + ['429', 429, 'RATE_LIMIT'], + ['503', 503, 'TRANSIENT'], + ])('drains the response body on a %s error', async (_status, statusCode, kind) => { + const cancel = vi.fn() + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue({ + status: statusCode, + ok: false, + json: async () => ({}), + body: { cancel }, + }), + ) + expect(await fetchPackagistPackageList()).toMatchObject({ kind }) + expect(cancel).toHaveBeenCalled() + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts new file mode 100644 index 0000000000..0cbead5fb6 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/normalize.test.ts @@ -0,0 +1,285 @@ +import { describe, expect, it } from 'vitest' + +import { + buildPackagistVersionRows, + extractVersionDependencies, + isPackagistDevVersion, + isPackagistPrerelease, + normalizePackagistStats, + packagistNameFromPurl, +} from '../normalize' +import type { PackagistPackageInfo } from '../types' + +describe('purl helpers', () => { + it('strips the pkg:composer prefix to recover vendor/name', () => { + expect(packagistNameFromPurl('pkg:composer/monolog/monolog')).toBe('monolog/monolog') + expect(packagistNameFromPurl('pkg:composer/symfony/http-kernel')).toBe('symfony/http-kernel') + }) +}) + +// B2 — dynamic-endpoint normalization. +describe('normalizePackagistStats', () => { + const full: PackagistPackageInfo = { + name: 'monolog/monolog', + description: 'Sends your logs to files and web services', + time: '2011-02-15T15:11:52+00:00', + maintainers: [{ name: 'seldaek', avatar_url: 'https://example.org/a.png' }, {}], + type: 'library', + repository: 'https://github.com/Seldaek/monolog', + language: 'PHP', + dependents: 5423, + suggesters: 300, + favers: 10000, + downloads: { total: 500_000_000, monthly: 10_000_000, daily: 350_000 }, + } + + it('extracts description, repository, dependents and total downloads', () => { + const stats = normalizePackagistStats(full) + expect(stats).toEqual({ + name: 'monolog/monolog', + description: 'Sends your logs to files and web services', + repositoryUrl: 'https://github.com/Seldaek/monolog', + status: 'active', + dependents: 5423, + downloadsTotal: 500_000_000, + // downloads.monthly is deliberately not extracted — packages.downloads_last_30d + // belongs exclusively to the dedicated downloads-30d lane + // nameless maintainer entries are dropped + maintainers: [{ username: 'seldaek', displayName: null, email: null, role: 'maintainer' }], + }) + }) + + it('maps abandoned → deprecated for both the boolean and replacement-name forms', () => { + expect(normalizePackagistStats({ ...full, abandoned: true }).status).toBe('deprecated') + expect(normalizePackagistStats({ ...full, abandoned: 'new/package' }).status).toBe('deprecated') + expect(normalizePackagistStats({ ...full, abandoned: false }).status).toBe('active') + expect(normalizePackagistStats(full).status).toBe('active') + }) + + it('drops a null/non-object maintainer entry instead of throwing', () => { + // isPackagistStatsJson only guarantees maintainers is an array, not that every + // element is a well-formed object — a rogue null must not crash `m.name` access. + const stats = normalizePackagistStats({ + name: 'a/b', + maintainers: [null, 'not-an-object', { name: 'seldaek' }] as never, + }) + expect(stats.maintainers).toEqual([ + { username: 'seldaek', displayName: null, email: null, role: 'maintainer' }, + ]) + }) + + it('tolerates missing optional fields with nulls and empty lists', () => { + const stats = normalizePackagistStats({ name: 'a/b' }) + expect(stats).toEqual({ + name: 'a/b', + description: null, + repositoryUrl: null, + status: 'active', + dependents: null, + downloadsTotal: null, + maintainers: [], + }) + }) +}) + +describe('isPackagistDevVersion', () => { + it('detects dev- branch versions', () => { + expect(isPackagistDevVersion('dev-main')).toBe(true) + expect(isPackagistDevVersion('dev-feature/foo')).toBe(true) + }) + + it('detects numbered branch versions via the -dev normalized suffix', () => { + expect(isPackagistDevVersion('2.x-dev', '2.9999999.9999999.9999999-dev')).toBe(true) + expect(isPackagistDevVersion('1.0.x-dev', '1.0.9999999.9999999-dev')).toBe(true) + }) + + it('keeps tagged releases', () => { + expect(isPackagistDevVersion('1.0.0', '1.0.0.0')).toBe(false) + expect(isPackagistDevVersion('v2.3.4', '2.3.4.0')).toBe(false) + }) +}) + +describe('isPackagistPrerelease', () => { + it('flags alpha/beta/RC suffixes', () => { + expect(isPackagistPrerelease('1.0.0.0-alpha2')).toBe(true) + expect(isPackagistPrerelease('1.0.0.0-beta1')).toBe(true) + expect(isPackagistPrerelease('1.0.0.0-RC1')).toBe(true) + }) + + it('treats plain and patch-suffixed versions as stable', () => { + expect(isPackagistPrerelease('1.0.0.0')).toBe(false) + expect(isPackagistPrerelease('1.0.0.0-patch1')).toBe(false) + }) +}) + +// C2 — version rows: dev branches skipped, latest by normalized numeric compare, +// stable preferred over prerelease, aggregates span the kept rows. +describe('buildPackagistVersionRows', () => { + it('skips dev branches and derives rows and aggregates from tagged releases', () => { + const { versionRows, latestVersion, firstReleaseAt, latestReleaseAt, licenses, homepage } = + buildPackagistVersionRows([ + { version: 'dev-main', version_normalized: 'dev-main' }, + { + version: '2.10.0', + version_normalized: '2.10.0.0', + time: '2024-03-01T00:00:00+00:00', + license: ['MIT'], + homepage: 'https://example.org', + }, + { + version: '2.9.1', + version_normalized: '2.9.1.0', + time: '2023-01-01T00:00:00+00:00', + license: ['MIT'], + }, + { + version: '1.0.0', + version_normalized: '1.0.0.0', + time: '2020-06-01T00:00:00+00:00', + license: ['BSD-3-Clause'], + homepage: 'https://old.example.org', + }, + ]) + + expect(versionRows).toHaveLength(3) + const byNumber = Object.fromEntries(versionRows.map((r) => [r.number, r])) + // numeric compare: 2.10.0 > 2.9.1 + expect(byNumber['2.10.0']).toEqual({ + number: '2.10.0', + publishedAt: '2024-03-01T00:00:00+00:00', + isLatest: true, + isPrerelease: false, + licenses: ['MIT'], + }) + expect(byNumber['2.9.1'].isLatest).toBe(false) + expect(byNumber['1.0.0'].licenses).toEqual(['BSD-3-Clause']) + expect(latestVersion).toBe('2.10.0') + expect(firstReleaseAt).toBe('2020-06-01T00:00:00+00:00') + expect(latestReleaseAt).toBe('2024-03-01T00:00:00+00:00') + // package-level licenses and homepage come from the latest version + expect(licenses).toEqual(['MIT']) + expect(homepage).toBe('https://example.org') + }) + + it('sorts a multi-license array deterministically, regardless of registry order', () => { + // versions.licenses is documented as deterministically sorted (schema comment) — + // Composer's dual-licensing means the registry can report the same license set in + // different orders across fetches, which must not register as a real change. + const { versionRows, licenses } = buildPackagistVersionRows([ + { + version: '1.0.0', + version_normalized: '1.0.0.0', + license: ['MIT', 'Apache-2.0', 'BSD-3-Clause'], + }, + ]) + expect(versionRows[0].licenses).toEqual(['Apache-2.0', 'BSD-3-Clause', 'MIT']) + expect(licenses).toEqual(['Apache-2.0', 'BSD-3-Clause', 'MIT']) + }) + + it('returns a null homepage when the latest version omits or blanks it', () => { + const absent = buildPackagistVersionRows([{ version: '1.0.0', version_normalized: '1.0.0.0' }]) + expect(absent.homepage).toBeNull() + + const blank = buildPackagistVersionRows([ + { version: '1.0.0', version_normalized: '1.0.0.0', homepage: ' ' }, + ]) + expect(blank.homepage).toBeNull() + }) + + it('prefers a stable release over a newer prerelease for latest', () => { + const { latestVersion, versionRows } = buildPackagistVersionRows([ + { + version: '3.0.0-RC1', + version_normalized: '3.0.0.0-RC1', + time: '2026-01-01T00:00:00+00:00', + }, + { version: '2.5.0', version_normalized: '2.5.0.0', time: '2025-01-01T00:00:00+00:00' }, + ]) + expect(latestVersion).toBe('2.5.0') + const byNumber = Object.fromEntries(versionRows.map((r) => [r.number, r])) + expect(byNumber['3.0.0-RC1'].isPrerelease).toBe(true) + expect(byNumber['3.0.0-RC1'].isLatest).toBe(false) + expect(byNumber['2.5.0'].isLatest).toBe(true) + }) + + it('falls back to the highest prerelease when no stable release exists', () => { + const { latestVersion } = buildPackagistVersionRows([ + { version: '1.0.0-beta2', version_normalized: '1.0.0.0-beta2' }, + { version: '1.0.0-alpha3', version_normalized: '1.0.0.0-alpha3' }, + ]) + expect(latestVersion).toBe('1.0.0-beta2') + }) + + it('picks the higher base version among prereleases regardless of suffix rank', () => { + const { latestVersion } = buildPackagistVersionRows([ + { version: '1.0.0-rc1', version_normalized: '1.0.0.0-rc1' }, + { version: '2.0.0-alpha1', version_normalized: '2.0.0.0-alpha1' }, + ]) + expect(latestVersion).toBe('2.0.0-alpha1') + }) + + it('breaks same-base prerelease ties by suffix rank, not array order', () => { + // lower-precedence suffix listed FIRST — array order must not decide + const { latestVersion } = buildPackagistVersionRows([ + { version: '1.0.0-alpha3', version_normalized: '1.0.0.0-alpha3' }, + { version: '1.0.0-beta2', version_normalized: '1.0.0.0-beta2' }, + ]) + expect(latestVersion).toBe('1.0.0-beta2') + }) + + it('distinguishes numbered suffixes of the same rank', () => { + const { latestVersion } = buildPackagistVersionRows([ + { version: '1.0.0-rc1', version_normalized: '1.0.0.0-rc1' }, + { version: '1.0.0-rc2', version_normalized: '1.0.0.0-rc2' }, + ]) + expect(latestVersion).toBe('1.0.0-rc2') + }) + + it('returns empty rows and null aggregates when only dev branches exist', () => { + expect( + buildPackagistVersionRows([{ version: 'dev-main', version_normalized: 'dev-main' }]), + ).toEqual({ + versionRows: [], + latestVersion: null, + firstReleaseAt: null, + latestReleaseAt: null, + licenses: null, + homepage: null, + }) + }) +}) + +// C3 — dependency extraction: require→direct, require-dev→dev, platform packages excluded. +describe('extractVersionDependencies', () => { + it('maps require to direct and require-dev to dev, preserving constraints', () => { + const deps = extractVersionDependencies({ + version: '2.0.0', + require: { php: '>=8.1', 'psr/log': '^2.0 || ^3.0' }, + 'require-dev': { 'phpunit/phpunit': '^10.5' }, + }) + expect(deps).toEqual([ + { name: 'psr/log', constraint: '^2.0 || ^3.0', kind: 'direct' }, + { name: 'phpunit/phpunit', constraint: '^10.5', kind: 'dev' }, + ]) + }) + + it('excludes platform packages (php, hhvm, composer-*, ext-*, lib-*)', () => { + const deps = extractVersionDependencies({ + version: '1.0.0', + require: { + php: '>=7.4', + hhvm: '*', + 'composer-plugin-api': '^2.0', + 'composer-runtime-api': '^2.2', + 'ext-json': '*', + 'lib-icu': '*', + 'real/dep': '^1.0', + }, + }) + expect(deps).toEqual([{ name: 'real/dep', constraint: '^1.0', kind: 'direct' }]) + }) + + it('returns [] when the version declares no dependencies', () => { + expect(extractVersionDependencies({ version: '1.0.0' })).toEqual([]) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts new file mode 100644 index 0000000000..1665157b83 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/persistMetadata.test.ts @@ -0,0 +1,217 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getPackagistPackageIdsByNames, + logAuditFieldChanges, + reconcileVersionDependencies, + updatePackagistVersionAggregates, + upsertPackagistVersions, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import type { PackagistExpandedVersion } from '../types' +import { persistPackagistMetadata } from '../upsertMetadata' + +vi.mock('@crowd/data-access-layer/src/packages', () => ({ + updatePackagistVersionAggregates: vi.fn(), + upsertPackagistVersions: vi.fn(), + getPackagistPackageIdsByNames: vi.fn(), + reconcileVersionDependencies: vi.fn().mockResolvedValue([]), + logAuditFieldChanges: vi.fn(), +})) + +const mockAggregates = vi.mocked(updatePackagistVersionAggregates) +const mockVersions = vi.mocked(upsertPackagistVersions) +const mockIds = vi.mocked(getPackagistPackageIdsByNames) +const mockDeps = vi.mocked(reconcileVersionDependencies) +const mockAudit = vi.mocked(logAuditFieldChanges) + +const qx = { + tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), +} as unknown as QueryExecutor +const PURL = 'pkg:composer/monolog/monolog' + +const expanded: PackagistExpandedVersion[] = [ + { + version: '2.0.0', + version_normalized: '2.0.0.0', + time: '2024-01-01T00:00:00+00:00', + license: ['MIT'], + homepage: 'https://monolog.example.org', + require: { php: '>=8.1', 'psr/log': '^3.0' }, + 'require-dev': { 'phpunit/phpunit': '^10.5' }, + }, + { + version: '1.0.0', + version_normalized: '1.0.0.0', + time: '2020-01-01T00:00:00+00:00', + license: ['MIT'], + require: { 'psr/log': '^1.0' }, + }, + { version: 'dev-main', version_normalized: 'dev-main' }, +] + +beforeEach(() => { + vi.clearAllMocks() + mockDeps.mockResolvedValue([]) +}) + +// C2 + C3 — persistence wiring: aggregates on packages, version rows, dependency edges +// resolved to existing packages rows only. +describe('persistPackagistMetadata', () => { + it('upserts version rows (dev branches excluded), aggregates, and resolved dependency edges', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: ['packages.latest_version'] }) + mockVersions.mockResolvedValue({ + changedFields: ['versions.number'], + versionIds: [ + { number: '2.0.0', id: '21' }, + { number: '1.0.0', id: '20' }, + ], + }) + mockIds.mockResolvedValue( + new Map([ + ['psr/log', '44'], + ['phpunit/phpunit', '45'], + ]), + ) + mockDeps.mockResolvedValue(['package_dependencies.depends_on_id']) + + const result = await persistPackagistMetadata(qx, PURL, expanded) + + expect(mockAggregates).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ + versionsCount: 2, + latestVersion: '2.0.0', + firstReleaseAt: '2020-01-01T00:00:00+00:00', + latestReleaseAt: '2024-01-01T00:00:00+00:00', + licenses: ['MIT'], + homepage: 'https://monolog.example.org', + }), + ) + expect(mockVersions).toHaveBeenCalledWith( + qx, + '9', + expect.arrayContaining([ + expect.objectContaining({ number: '2.0.0', isLatest: true }), + expect.objectContaining({ number: '1.0.0', isLatest: false }), + ]), + '2.0.0', + ) + expect(mockVersions.mock.calls[0][2]).toHaveLength(2) + + // platform 'php' never reaches resolution + expect(mockIds).toHaveBeenCalledTimes(1) + const requestedNames = mockIds.mock.calls[0][1] + expect([...requestedNames].sort()).toEqual(['phpunit/phpunit', 'psr/log']) + + expect(mockDeps).toHaveBeenCalledTimes(1) + // reconciliation is scoped to exactly the versions refreshed this pass + expect(mockDeps.mock.calls[0][1]).toEqual(expect.arrayContaining(['21', '20'])) + const edges = mockDeps.mock.calls[0][2] + expect(edges).toEqual( + expect.arrayContaining([ + { packageId: '9', versionId: '21', dependsOnId: '44', constraint: '^3.0', kind: 'direct' }, + { packageId: '9', versionId: '21', dependsOnId: '45', constraint: '^10.5', kind: 'dev' }, + { packageId: '9', versionId: '20', dependsOnId: '44', constraint: '^1.0', kind: 'direct' }, + ]), + ) + expect(edges).toHaveLength(3) + + expect(result.found).toBe(true) + expect(result.unresolvedDependencyTargets).toBe(0) + // dependency-edge changes must reach the audit log alongside aggregates/versions + expect(result.changedFields).toContain('package_dependencies.depends_on_id') + // audited atomically inside the same transaction as the writes above + expect(mockAudit).toHaveBeenCalledWith( + qx, + 'packagist', + PURL, + expect.arrayContaining(['packages.latest_version', 'package_dependencies.depends_on_id']), + ) + }) + + it('skips and counts dependency targets that do not resolve to a packages row', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: [] }) + mockVersions.mockResolvedValue({ + changedFields: [], + versionIds: [ + { number: '2.0.0', id: '21' }, + { number: '1.0.0', id: '20' }, + ], + }) + // phpunit/phpunit is unknown + mockIds.mockResolvedValue(new Map([['psr/log', '44']])) + + const result = await persistPackagistMetadata(qx, PURL, expanded) + + const edges = mockDeps.mock.calls[0][2] + expect(edges).toHaveLength(2) + expect(edges.every((e) => e.dependsOnId === '44')).toBe(true) + expect(result.unresolvedDependencyTargets).toBe(1) + }) + + it('reconciles (deletes stale edges) even when a version now declares zero dependencies', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: [] }) + mockVersions.mockResolvedValue({ + changedFields: [], + versionIds: [{ number: '1.0.0', id: '20' }], + }) + mockDeps.mockResolvedValue(['package_dependencies.depends_on_id']) + + // no require/require-dev at all — a prior run's edges for this version must still + // be reconciled away, not left stale just because nothing new was declared + const result = await persistPackagistMetadata(qx, PURL, [ + { version: '1.0.0', version_normalized: '1.0.0.0' }, + ]) + + expect(mockIds).not.toHaveBeenCalled() + expect(mockDeps).toHaveBeenCalledWith(qx, ['20'], []) + expect(result.changedFields).toContain('package_dependencies.depends_on_id') + }) + + it('returns found=false and writes nothing when the packages row is missing', async () => { + mockAggregates.mockResolvedValue(null) + + const result = await persistPackagistMetadata(qx, PURL, expanded) + + expect(result).toEqual({ found: false, changedFields: [], unresolvedDependencyTargets: 0 }) + expect(mockVersions).not.toHaveBeenCalled() + expect(mockIds).not.toHaveBeenCalled() + expect(mockDeps).not.toHaveBeenCalled() + expect(mockAudit).not.toHaveBeenCalled() + }) + + it('handles a dev-branches-only package: aggregates written, no version or dependency writes', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: [] }) + + const result = await persistPackagistMetadata(qx, PURL, [ + { version: 'dev-main', version_normalized: 'dev-main' }, + ]) + + expect(mockAggregates).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ versionsCount: 0, latestVersion: null, homepage: null }), + ) + expect(mockVersions).not.toHaveBeenCalled() + expect(mockDeps).not.toHaveBeenCalled() + expect(result.found).toBe(true) + }) + + it('strips NUL bytes from the homepage before writing (Postgres rejects them)', async () => { + mockAggregates.mockResolvedValue({ id: '9', changedFields: [] }) + mockVersions.mockResolvedValue({ changedFields: [], versionIds: [] }) + + await persistPackagistMetadata(qx, PURL, [ + { version: '1.0.0', version_normalized: '1.0.0.0', homepage: 'https://ex\0ample.org' }, + ]) + + expect(mockAggregates).toHaveBeenCalledWith( + qx, + PURL, + expect.objectContaining({ homepage: 'https://example.org' }), + ) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts new file mode 100644 index 0000000000..69bbacdd98 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/persistPackageInfo.test.ts @@ -0,0 +1,217 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { + getOrCreateRepoByUrl, + logAuditFieldChanges, + removeDeclaredPackageRepo, + updatePackagistPackageStats, + upsertPackageMaintainers, + upsertPackageRepoPreserveProvenance, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import type { NormalizedPackagistStats } from '../types' +import { persistPackagistPackageInfo } from '../upsertPackageInfo' + +vi.mock('@crowd/data-access-layer/src/packages', () => ({ + updatePackagistPackageStats: vi.fn(), + upsertPackageMaintainers: vi.fn().mockResolvedValue([]), + getOrCreateRepoByUrl: vi.fn(), + upsertPackageRepoPreserveProvenance: vi.fn().mockResolvedValue([]), + removeDeclaredPackageRepo: vi.fn().mockResolvedValue([]), + logAuditFieldChanges: vi.fn(), +})) + +const mockUpdate = vi.mocked(updatePackagistPackageStats) +const mockMaintainers = vi.mocked(upsertPackageMaintainers) +const mockRepoGet = vi.mocked(getOrCreateRepoByUrl) +const mockRepoLink = vi.mocked(upsertPackageRepoPreserveProvenance) +const mockRepoRemove = vi.mocked(removeDeclaredPackageRepo) +const mockAudit = vi.mocked(logAuditFieldChanges) + +const qx = { + tx: vi.fn((cb: (t: QueryExecutor) => Promise) => cb(qx)), +} as unknown as QueryExecutor +const PURL = 'pkg:composer/monolog/monolog' + +const stats: NormalizedPackagistStats = { + name: 'monolog/monolog', + description: 'logs', + repositoryUrl: 'https://github.com/Seldaek/monolog', + status: 'active', + dependents: 42, + downloadsTotal: 1000, + maintainers: [{ username: 'seldaek', displayName: null, email: null, role: 'maintainer' }], +} + +beforeEach(() => { + vi.clearAllMocks() + mockRepoGet.mockResolvedValue({ id: '55', changedFields: [] }) +}) + +// Dynamic-endpoint persistence inside the metadata lane: packages fields + repo link for +// ALL packages; maintainers only for critical ones. Download rows are NOT written here — +// they belong to the dedicated downloads-30d/daily lanes. +describe('persistPackagistPackageInfo', () => { + it('updates the packages row, links the repo, and writes maintainers for a critical package', async () => { + mockUpdate.mockResolvedValue({ + id: '7', + isCritical: true, + changedFields: ['packages.description'], + }) + mockMaintainers.mockResolvedValue(['maintainers.display_name']) + + const result = await persistPackagistPackageInfo(qx, PURL, stats) + + expect(mockUpdate).toHaveBeenCalledWith( + qx, + expect.objectContaining({ + purl: PURL, + description: 'logs', + status: 'active', + totalDownloads: 1000, + dependentCount: 42, + }), + ) + // packages.downloads_last_30d belongs exclusively to the dedicated downloads-30d + // lane's boundary-anchored snapshot — the metadata lane must never touch it + expect(mockUpdate.mock.calls[0][1]).not.toHaveProperty('downloadsLast30d') + // canonicalized (lowercased) url + coarse host, linked with the manifest-declared convention + expect(mockRepoGet).toHaveBeenCalledWith(qx, 'https://github.com/seldaek/monolog', 'github') + expect(mockRepoLink).toHaveBeenCalledWith(qx, '7', '55', 'declared', 0.8) + // any stale 'declared' link pointing at a different repo is pruned in the same pass + expect(mockRepoRemove).toHaveBeenCalledWith(qx, '7', '55') + expect(mockMaintainers).toHaveBeenCalledWith(qx, '7', stats.maintainers, 'packagist') + expect(result.found).toBe(true) + expect(result.changedFields).toContain('packages.description') + // maintainer changes must reach the audit log too, matching the npm/pypi paths + expect(result.changedFields).toContain('maintainers.display_name') + // audited atomically inside the same transaction as the writes above + expect(mockAudit).toHaveBeenCalledWith( + qx, + 'packagist', + PURL, + expect.arrayContaining(['packages.description', 'maintainers.display_name']), + ) + }) + + it('skips maintainers for a non-critical package but still links the repo', async () => { + mockUpdate.mockResolvedValue({ id: '8', isCritical: false, changedFields: [] }) + + await persistPackagistPackageInfo(qx, PURL, stats) + + expect(mockMaintainers).not.toHaveBeenCalled() + expect(mockRepoGet).toHaveBeenCalledWith(qx, 'https://github.com/seldaek/monolog', 'github') + expect(mockRepoLink).toHaveBeenCalledWith(qx, '8', '55', 'declared', 0.8) + }) + + it('prunes a stale declared link when the repository URL switches to a different repo', async () => { + mockUpdate.mockResolvedValue({ id: '7', isCritical: false, changedFields: [] }) + mockRepoGet.mockResolvedValue({ id: '99', changedFields: [] }) + mockRepoRemove.mockResolvedValue(['package_repos.repo_id']) + + const result = await persistPackagistPackageInfo(qx, PURL, stats) + + expect(mockRepoLink).toHaveBeenCalledWith(qx, '7', '99', 'declared', 0.8) + // old link (some other repo_id) removed, new one (99) kept + expect(mockRepoRemove).toHaveBeenCalledWith(qx, '7', '99') + expect(result.changedFields).toContain('package_repos.repo_id') + }) + + it('reconciles a maintainer list that dropped to empty for a critical package', async () => { + mockUpdate.mockResolvedValue({ id: '7', isCritical: true, changedFields: [] }) + mockMaintainers.mockResolvedValue(['package_maintainers.maintainer_id']) + + const result = await persistPackagistPackageInfo(qx, PURL, { ...stats, maintainers: [] }) + + // upsertPackageMaintainers has replace/delete semantics — it must still run with an + // empty list so stale rows for maintainers no longer reported get removed. + expect(mockMaintainers).toHaveBeenCalledWith(qx, '7', [], 'packagist') + expect(result.changedFields).toContain('package_maintainers.maintainer_id') + }) + + it('skips the repo link and clears any previously-declared one when there is no repository URL', async () => { + mockUpdate.mockResolvedValue({ id: '7', isCritical: true, changedFields: [] }) + mockRepoRemove.mockResolvedValue(['package_repos.repo_id']) + + const result = await persistPackagistPackageInfo(qx, PURL, { ...stats, repositoryUrl: null }) + + expect(mockUpdate).toHaveBeenCalledWith(qx, expect.objectContaining({ repositoryUrl: null })) + expect(mockRepoGet).not.toHaveBeenCalled() + expect(mockRepoLink).not.toHaveBeenCalled() + expect(mockRepoRemove).toHaveBeenCalledWith(qx, '7') + expect(result.changedFields).toContain('package_repos.repo_id') + }) + + it('skips the repo link and clears the stale one when the repository URL cannot be canonicalized', async () => { + mockUpdate.mockResolvedValue({ id: '7', isCritical: true, changedFields: [] }) + + await persistPackagistPackageInfo(qx, PURL, { ...stats, repositoryUrl: 'not-a-valid-url' }) + + expect(mockRepoGet).not.toHaveBeenCalled() + expect(mockRepoLink).not.toHaveBeenCalled() + expect(mockRepoRemove).toHaveBeenCalledWith(qx, '7') + }) + + it('does not trust a canonicalized host outside the SCM allowlist (wiki/issue-tracker/registry URLs)', async () => { + mockUpdate.mockResolvedValue({ id: '7', isCritical: true, changedFields: [] }) + + await persistPackagistPackageInfo(qx, PURL, { + ...stats, + // canonicalizeRepoUrl resolves this to a real { url, host: 'other' } pair — + // it's not a parse failure, just not a verified SCM host. + repositoryUrl: 'https://www.mediawiki.org/wiki/Extension:Nuke', + }) + + expect(mockUpdate).toHaveBeenCalledWith(qx, expect.objectContaining({ repositoryUrl: null })) + expect(mockRepoGet).not.toHaveBeenCalled() + expect(mockRepoLink).not.toHaveBeenCalled() + expect(mockRepoRemove).toHaveBeenCalledWith(qx, '7') + }) + + it('folds repo changed-fields into the result for the audit log', async () => { + mockUpdate.mockResolvedValue({ + id: '7', + isCritical: false, + changedFields: ['packages.description'], + }) + mockRepoGet.mockResolvedValue({ id: '55', changedFields: ['repos.url', 'repos.host'] }) + mockRepoLink.mockResolvedValue(['package_repos.repo_id']) + + const result = await persistPackagistPackageInfo(qx, PURL, stats) + + expect(result.changedFields).toEqual( + expect.arrayContaining([ + 'packages.description', + 'repos.url', + 'repos.host', + 'package_repos.repo_id', + ]), + ) + }) + + it('returns found=false and writes nothing else when the packages row is missing', async () => { + mockUpdate.mockResolvedValue(null) + + const result = await persistPackagistPackageInfo(qx, PURL, stats) + + expect(result).toEqual({ found: false, changedFields: [] }) + expect(mockRepoGet).not.toHaveBeenCalled() + expect(mockMaintainers).not.toHaveBeenCalled() + expect(mockAudit).not.toHaveBeenCalled() + }) + + it('strips NUL bytes from the description before writing (Postgres rejects them)', async () => { + mockUpdate.mockResolvedValue({ id: '7', isCritical: false, changedFields: [] }) + + await persistPackagistPackageInfo(qx, PURL, { + ...stats, + description: 'Esta es una descripcin random', + }) + + expect(mockUpdate).toHaveBeenCalledWith( + qx, + expect.objectContaining({ description: 'Esta es una descripcin random' }), + ) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/__tests__/wiring.test.ts b/services/apps/packages_worker/src/packagist/__tests__/wiring.test.ts new file mode 100644 index 0000000000..dda12c6401 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/__tests__/wiring.test.ts @@ -0,0 +1,49 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' + +import { FETCHERS } from '../../security-contacts/extractors/registry/index' +import { PACKAGIST_CRONS } from '../schedule' + +// D1 — wiring that unit tests can meaningfully pin down. + +describe('package.json worker scripts', () => { + // vitest runs with cwd at the package root; import.meta is unavailable under the CJS tsconfig. + const pkg = JSON.parse(readFileSync('package.json', 'utf8')) as { + scripts: Record + } + + it('declares start/dev scripts for the packagist worker', () => { + expect(pkg.scripts['start:packagist-worker']).toContain('src/bin/packagist-worker.ts') + expect(pkg.scripts['dev:packagist-worker']).toContain('src/bin/packagist-worker.ts') + }) +}) + +describe('schedule cadence', () => { + it('defines the three packagist crons with minutes off :00 (crawler guideline)', () => { + // metadata has no cron — the seed workflow chains it as a child on completion + const crons = Object.entries(PACKAGIST_CRONS) + expect(crons.map(([name]) => name).sort()).toEqual(['downloads30d', 'downloadsDaily', 'seed']) + for (const [name, cron] of crons) { + const minute = cron.split(' ')[0] + expect(minute, `${name} cron minute`).toMatch(/^[1-9][0-9]?$/) + expect(Number(minute), `${name} cron minute`).toBeLessThan(60) + } + }) + + it('runs seed weekly, the 30d window capture monthly on the 1st, and daily downloads daily', () => { + expect(PACKAGIST_CRONS.seed.split(' ')).toHaveLength(5) + expect(PACKAGIST_CRONS.seed.split(' ')[4]).not.toBe('*') + // monthly, anchored on the 1st so the observed rolling value sits on the boundary + expect(PACKAGIST_CRONS.downloads30d.split(' ')[2]).toBe('1') + expect(PACKAGIST_CRONS.downloads30d.split(' ')[4]).toBe('*') + expect(PACKAGIST_CRONS.downloadsDaily.split(' ')[2]).toBe('*') + expect(PACKAGIST_CRONS.downloadsDaily.split(' ')[4]).toBe('*') + }) +}) + +describe('security-contacts registry fetchers', () => { + it('keys the Composer fetcher by the packagist ecosystem value', () => { + expect(FETCHERS.packagist).toBeTypeOf('function') + expect('composer' in FETCHERS).toBe(false) + }) +}) diff --git a/services/apps/packages_worker/src/packagist/activities.ts b/services/apps/packages_worker/src/packagist/activities.ts new file mode 100644 index 0000000000..ee610e26df --- /dev/null +++ b/services/apps/packages_worker/src/packagist/activities.ts @@ -0,0 +1,468 @@ +import { Context } from '@temporalio/activity' + +import { partition, timeout } from '@crowd/common' +import { + getCriticalPackagistPackageCount, + getPackagist30dDuePurls, + getPackagistDailyDownloadsDue, + getPackagistMetadataDuePurls, + insertDailyDownloads, + insertPackagistPackages, + logAuditFieldChanges, + markPackagist30dProcessed, + markPackagistDailyProcessed, + markPackagistMetadataScanned, +} from '@crowd/data-access-layer/src/packages' +import type { + PackagistDailyCandidate, + PackagistMetadataCandidate, + PackagistRunResult, +} from '@crowd/data-access-layer/src/packages/packagistPackageState' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { getServiceChildLogger } from '@crowd/logging' + +import { getPackagesDb } from '../db' +import { mapWithConcurrency } from '../utils/concurrency' +import { isClientError } from '../utils/isClientError' + +import { persistPackagist30dWindow } from './downloads' +import { expandComposerMetadata } from './expandMetadata' +import { fetchPackagistP2, fetchPackagistStats } from './fetchPackage' +import { fetchPackagistPackageList, parsePackagistPackageList } from './listPackages' +import { normalizePackagistStats, packagistNameFromPurl } from './normalize' +import { INGEST_MAX_ATTEMPTS } from './retryPolicy' +import { FetchError, isFetchError, isP2NotModified } from './types' +import { persistPackagistMetadata } from './upsertMetadata' +import { persistPackagistPackageInfo } from './upsertPackageInfo' + +const log = getServiceChildLogger('packagist') + +const WORKER = 'packagist' + +// 4xx/malformed get a few quick in-lane retries with a small linear backoff +const INGEST_4XX_ATTEMPTS = 3 +const INGEST_4XX_BACKOFF_MS = 1000 + +// Concurrency cap for the dynamic (packagist.org) endpoint, shared by all lanes +// that fetch it: metadata, downloads-30d, and daily downloads. +function statsConcurrency(): number { + const n = parseInt(process.env.CROWD_PACKAGES_PACKAGIST_STATS_CONCURRENCY ?? '10', 10) + return Math.max(1, Math.min(10, Number.isFinite(n) ? n : 10)) +} + +function metadataRefreshDays(): number { + const n = parseInt(process.env.CROWD_PACKAGES_PACKAGIST_METADATA_REFRESH_DAYS ?? '7', 10) + return Number.isFinite(n) && n > 0 ? n : 7 +} + +// Scope of the metadata sweep. Deliberately diverges from pypi (critical-only steady +// state): Packagist enriches ALL packages because deps.dev has no Packagist data to +// fall back on. Set CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL=true to narrow +// back to the critical slice. +function runOnlyForCritical(): boolean { + const raw = (process.env.CROWD_PACKAGES_PACKAGIST_RUN_ONLY_FOR_CRITICAL ?? 'false') + .trim() + .toLowerCase() + return raw === 'true' || raw === '1' || raw === 'yes' +} + +export async function packagistStopAfterFirstPage(): Promise { + const raw = (process.env.CROWD_PACKAGES_PACKAGIST_STOP_AFTER_FIRST_PAGE ?? 'false') + .trim() + .toLowerCase() + return raw === 'true' || raw === '1' || raw === 'yes' +} + +// Deterministic cutoff source for the watermark-draining download workflows. +export async function packagistCurrentTimestamp(): Promise { + return new Date().toISOString() +} + +// Fetch with the shared fast-retry contract: transient/429 results throw so Temporal +// retries the batch; 4xx/malformed get INGEST_4XX_ATTEMPTS quick in-lane retries and +// then surface as a give-up `error` the caller records on the state row. +async function fetchWithFastRetry( + fetchOnce: () => Promise, + what: string, +): Promise<{ value: T; attempts: number } | { error: FetchError; attempts: number }> { + for (let attempt = 1; ; attempt++) { + const result = await fetchOnce() + + if (!isFetchError(result)) { + return { value: result, attempts: attempt } + } + + if (!isClientError(result.statusCode, result.kind) && result.kind !== 'MALFORMED') { + throw new Error(`Failed to fetch ${what}: ${result.message}`) + } + + if (attempt >= INGEST_4XX_ATTEMPTS) { + return { error: result, attempts: attempt } + } + + await timeout(attempt * INGEST_4XX_BACKOFF_MS) + } +} + +function giveUpResult(error: FetchError, attempts: number): PackagistRunResult { + return { + status: 'error', + attempts, + httpStatus: error.statusCode, + errorKind: error.kind, + message: error.message, + } +} + +// The merged enrichment lane: one pass per package fetches BOTH registry endpoints — +// the dynamic one (package info, repo link, maintainers) and p2 (versions, dependencies). +export async function ingestOnePackagistMetadata( + qx: QueryExecutor, + candidate: PackagistMetadataCandidate, + // This batch activity's own scheduledTimestampMs (stable across Temporal retries), + // passed through to every give-up write below — see MarkMetadataScannedOptions.notBefore. + scheduledAt: string, +): Promise { + const name = packagistNameFromPurl(candidate.purl) + + // Phase 1: dynamic endpoint + const info = await fetchWithFastRetry( + () => fetchPackagistStats(name), + `Packagist stats for ${name}`, + ) + if ('error' in info) { + log.warn( + { purl: candidate.purl, statusCode: info.error.statusCode, kind: info.error.kind }, + 'packagist package info 4xx/malformed after fast retries — marking scanned and skipping', + ) + await markPackagistMetadataScanned( + qx, + candidate.purl, + giveUpResult(info.error, info.attempts), + { + notBefore: scheduledAt, + }, + ) + return + } + + const stats = normalizePackagistStats(info.value.package) + // persistPackagistPackageInfo audits its own writes atomically, inside the same + // transaction — phase 1 is committed-and-audited before the p2 fetch (which can + // throw) ever runs. + await persistPackagistPackageInfo(qx, candidate.purl, stats) + + // Phase 2: p2 endpoint + const p2 = await fetchWithFastRetry( + () => fetchPackagistP2(name, candidate.metadataLastModified), + `Packagist metadata for ${name}`, + ) + if ('error' in p2) { + log.warn( + { purl: candidate.purl, statusCode: p2.error.statusCode, kind: p2.error.kind }, + 'packagist metadata 4xx/malformed after fast retries — marking scanned and skipping', + ) + // Phase 1 already succeeded — only p2 (versions/deps) failed to refresh. Don't push + // metadata_last_run_at forward, or due-selection wrongly treats this package as + // "recently scanned" and skips it for the full refresh window despite stale p2 data. + await markPackagistMetadataScanned(qx, candidate.purl, giveUpResult(p2.error, p2.attempts), { + bumpLastRunAt: false, + notBefore: scheduledAt, + }) + return + } + + let lastModified: string | null = null + if (!isP2NotModified(p2.value)) { + const expanded = expandComposerMetadata(p2.value.minifiedVersions) + // persistPackagistMetadata audits its own writes atomically, inside the same + // transaction as the aggregate/version/dependency writes. + const persistResult = await persistPackagistMetadata(qx, candidate.purl, expanded) + if (persistResult.unresolvedDependencyTargets > 0) { + log.debug( + { purl: candidate.purl, unresolved: persistResult.unresolvedDependencyTargets }, + 'packagist dependency targets not found in packages — edges skipped', + ) + } + lastModified = p2.value.lastModified + } + + await markPackagistMetadataScanned( + qx, + candidate.purl, + { status: 'success', attempts: p2.attempts }, + { + metadataLastModified: lastModified, + }, + ) +} + +// The monthly downloads-30d lane: dynamic fetch, one window row per purl per month. +export async function ingestOnePackagist30dWindow( + qx: QueryExecutor, + purl: string, + runDate: string, + scheduledAt: string, +): Promise { + const name = packagistNameFromPurl(purl) + + const info = await fetchWithFastRetry( + () => fetchPackagistStats(name), + `Packagist stats for ${name}`, + ) + if ('error' in info) { + log.warn( + { purl, statusCode: info.error.statusCode, kind: info.error.kind }, + 'packagist 30d downloads 4xx/malformed after fast retries — marking processed and skipping', + ) + await markPackagist30dProcessed(qx, purl, giveUpResult(info.error, info.attempts), scheduledAt) + return + } + + // persistPackagist30dWindow audits its own write atomically, inside the same + // transaction as the insert-if-absent. + await persistPackagist30dWindow(qx, purl, info.value.package.downloads?.monthly ?? null, runDate) + await markPackagist30dProcessed(qx, purl, { status: 'success', attempts: info.attempts }) +} + +// The daily downloads lane (critical slice): dynamic fetch, one downloads_daily row. +export async function ingestOnePackagistDailyDownload( + qx: QueryExecutor, + candidate: PackagistDailyCandidate, + runDate: string, + scheduledAt: string, +): Promise { + const name = packagistNameFromPurl(candidate.purl) + + const info = await fetchWithFastRetry( + () => fetchPackagistStats(name), + `Packagist stats for ${name}`, + ) + if ('error' in info) { + log.warn( + { purl: candidate.purl, statusCode: info.error.statusCode, kind: info.error.kind }, + 'packagist daily downloads 4xx/malformed after fast retries — marking processed and skipping', + ) + await markPackagistDailyProcessed( + qx, + candidate.purl, + giveUpResult(info.error, info.attempts), + scheduledAt, + ) + return + } + + const daily = info.value.package.downloads?.daily + if (typeof daily === 'number') { + // Insert + audit share one transaction so a failed audit insert can never leave a + // committed row unaudited — a retry would hit ON CONFLICT DO NOTHING and report no + // changes, permanently losing the audit event otherwise. + await qx.tx(async (t) => { + const changedFields = await insertDailyDownloads(t, candidate.packageId, [ + { day: runDate, downloads: daily }, + ]) + await logAuditFieldChanges(t, WORKER, candidate.purl, changedFields) + }) + } + await markPackagistDailyProcessed(qx, candidate.purl, { + status: 'success', + attempts: info.attempts, + }) +} + +export async function ingestPackagistItemsConcurrently( + items: T[], + attempt: number, + concurrency: number, + ingest: (item: T) => Promise, + onGiveUp: (item: T, err: unknown) => Promise, +): Promise { + // mapWithConcurrency stops scheduling new items after the first rejection from the + // wrapped callback. `attempt` tracks the whole batch activity, not each item, so a + // rethrow here on an early item would starve every later item in the array of a + // genuine try this round. Never rethrow from inside the callback — collect the first + // retryable failure and decide once, after every item has actually been attempted. + let firstRetryableError: unknown + + await mapWithConcurrency(items, concurrency, async (item) => { + try { + await ingest(item) + } catch (err) { + // Retry via Temporal while attempts remain; then give up and continue + if (attempt < INGEST_MAX_ATTEMPTS) { + if (firstRetryableError === undefined) firstRetryableError = err + return + } + log.warn( + { item: String(item), attempt, err: String(err) }, + 'packagist item failed after max attempts — giving up', + ) + await onGiveUp(item, err) + } + }) + + if (firstRetryableError !== undefined) throw firstRetryableError +} + +export async function runPackagistPackageSeed(): Promise<{ discovered: number; invalid: number }> { + const result = await fetchPackagistPackageList() + + if (isFetchError(result)) { + throw new Error(`Failed to fetch Packagist package list: ${result.message}`) + } + + const { entries, invalid } = parsePackagistPackageList(result) + + if (entries.length > 0) { + const qx = await getPackagesDb() + for (const chunk of partition(entries, 5000)) { + await insertPackagistPackages(qx, chunk) + } + } + + return { discovered: entries.length, invalid } +} + +// `cutoff` is the drain's own fixed start time (stable across every round via +// continueAsNew), not a live NOW() — a keyset scan only visits each purl once per +// drain, so re-deriving "7 days ago" fresh on every batch would silently skip a purl +// that hasn't quite hit the refresh window yet when the cursor passes it, pushing its +// effective cadence out toward two refresh cycles instead of one. +export async function getPackagistMetadataBatch( + cutoff: string, + afterPurl: string, + batchSize: number, +): Promise<{ candidates: PackagistMetadataCandidate[]; nextCursor: string }> { + const qx = await getPackagesDb() + const dueCutoff = new Date( + new Date(cutoff).getTime() - metadataRefreshDays() * 24 * 60 * 60 * 1000, + ).toISOString() + const candidates = await getPackagistMetadataDuePurls( + qx, + dueCutoff, + afterPurl, + batchSize, + runOnlyForCritical(), + ) + return { + candidates, + nextCursor: candidates.length ? candidates[candidates.length - 1].purl : afterPurl, + } +} + +export async function ingestPackagistMetadataBatch( + candidates: PackagistMetadataCandidate[], +): Promise { + if (candidates.length === 0) return + const qx = await getPackagesDb() + const attempt = Context.current().info.attempt + // Stable across every Temporal retry of this same batch — see notBefore below. + const scheduledAt = new Date(Context.current().info.scheduledTimestampMs).toISOString() + + // The merged lane starts every ingest with a DYNAMIC-endpoint fetch, so it is + // bounded by that endpoint's 10-concurrent limit — not p2's 20. Running hotter + // gets connections reset by packagist.org ("fetch failed"). + await ingestPackagistItemsConcurrently( + candidates, + attempt, + statsConcurrency(), + (candidate) => ingestOnePackagistMetadata(qx, candidate, scheduledAt), + (candidate, err) => + markPackagistMetadataScanned( + qx, + candidate.purl, + { status: 'error', attempts: attempt, message: String(err) }, + { + // An item that already succeeded earlier in this same batch's retry + // sequence must not have that success overwritten by an unrelated + // re-processing failure. + notBefore: scheduledAt, + // This generic catch-all fires whenever ingestOnePackagistMetadata threw + // (its own classified give-up paths return normally instead) — meaning we + // can't tell whether phase 1 alone succeeded before a transient p2 failure + // exhausted Temporal's retries. Never bump the refresh watermark here, or a + // genuine p2/versions failure gets hidden for the full refresh window. + bumpLastRunAt: false, + }, + ), + ) + + log.info({ count: candidates.length }, 'Ingested Packagist metadata batch') +} + +export async function getPackagist30dBatch( + cutoff: string, + afterPurl: string, + batchSize: number, +): Promise<{ purls: string[]; nextCursor: string }> { + const qx = await getPackagesDb() + const purls = await getPackagist30dDuePurls(qx, cutoff, afterPurl, batchSize) + return { purls, nextCursor: purls.length ? purls[purls.length - 1] : afterPurl } +} + +export async function ingestPackagist30dBatch(purls: string[], runDate: string): Promise { + if (purls.length === 0) return + const qx = await getPackagesDb() + const attempt = Context.current().info.attempt + const scheduledAt = new Date(Context.current().info.scheduledTimestampMs).toISOString() + + await ingestPackagistItemsConcurrently( + purls, + attempt, + statsConcurrency(), + (purl) => ingestOnePackagist30dWindow(qx, purl, runDate, scheduledAt), + (purl, err) => + markPackagist30dProcessed( + qx, + purl, + { status: 'error', attempts: attempt, message: String(err) }, + scheduledAt, + ), + ) + + log.info({ count: purls.length }, 'Ingested Packagist 30d downloads batch') +} + +export async function getPackagistDailyBatch( + cutoff: string, + afterPurl: string, + batchSize: number, +): Promise<{ candidates: PackagistDailyCandidate[]; nextCursor: string }> { + const qx = await getPackagesDb() + const candidates = await getPackagistDailyDownloadsDue(qx, cutoff, afterPurl, batchSize) + return { + candidates, + nextCursor: candidates.length ? candidates[candidates.length - 1].purl : afterPurl, + } +} + +export async function ingestPackagistDailyBatch( + candidates: PackagistDailyCandidate[], + runDate: string, +): Promise { + if (candidates.length === 0) return + const qx = await getPackagesDb() + const attempt = Context.current().info.attempt + const scheduledAt = new Date(Context.current().info.scheduledTimestampMs).toISOString() + + await ingestPackagistItemsConcurrently( + candidates, + attempt, + statsConcurrency(), + (candidate) => ingestOnePackagistDailyDownload(qx, candidate, runDate, scheduledAt), + (candidate, err) => + markPackagistDailyProcessed( + qx, + candidate.purl, + { status: 'error', attempts: attempt, message: String(err) }, + scheduledAt, + ), + ) + + log.info({ count: candidates.length }, 'Ingested Packagist daily downloads batch') +} + +export async function getCriticalPackagistCount(): Promise { + const qx = await getPackagesDb() + return getCriticalPackagistPackageCount(qx) +} diff --git a/services/apps/packages_worker/src/packagist/downloads.ts b/services/apps/packages_worker/src/packagist/downloads.ts new file mode 100644 index 0000000000..f8d609a218 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/downloads.ts @@ -0,0 +1,52 @@ +import { + insertLast30dDownloadIfAbsent, + logAuditFieldChanges, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +const WORKER = 'packagist' + +// Compute the monthly window for observed rolling 30d downloads. +// endDate = first of the run month; startDate = endDate minus 30 days. +export function monthlyWindowFor(runDate: string): { startDate: string; endDate: string } { + const d = new Date(runDate + 'T00:00:00Z') + // endDate: first of the current month + const endDate = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1)) + // startDate: endDate minus 30 days + const startDate = new Date(endDate.getTime() - 30 * 24 * 60 * 60 * 1000) + + return { + startDate: startDate.toISOString().slice(0, 10), + endDate: endDate.toISOString().slice(0, 10), + } +} + +// One window row per purl per month, mirrored to packages.downloads_last_30d +// (npm parity). A window already recorded for the month is never overwritten — +// the value is the observation closest to the boundary. insertLast30dDownloadIfAbsent +// does the presence check and the insert (+ mirror) atomically in one statement, so +// concurrent runs for the same purl+month can't both pass a check-then-insert gap and +// have the later one clobber the earlier one's count. The audit record shares the same +// transaction, so a failed audit insert can never leave a committed window unaudited. +export async function persistPackagist30dWindow( + qx: QueryExecutor, + purl: string, + monthly: number | null, + runDate: string, +): Promise { + if (monthly == null) return [] + + const { startDate, endDate } = monthlyWindowFor(runDate) + return qx.tx(async (t) => { + const changedFields = await insertLast30dDownloadIfAbsent( + t, + purl, + startDate, + endDate, + monthly, + true, + ) + await logAuditFieldChanges(t, WORKER, purl, changedFields) + return changedFields + }) +} diff --git a/services/apps/packages_worker/src/packagist/expandMetadata.ts b/services/apps/packages_worker/src/packagist/expandMetadata.ts new file mode 100644 index 0000000000..2acb74278e --- /dev/null +++ b/services/apps/packages_worker/src/packagist/expandMetadata.ts @@ -0,0 +1,29 @@ +import type { PackagistExpandedVersion, PackagistMinifiedVersion } from './types' + +export function expandComposerMetadata( + versions: PackagistMinifiedVersion[], +): PackagistExpandedVersion[] { + if (versions.length === 0) return [] + + const result: PackagistExpandedVersion[] = [] + let current: Record = {} + + for (const minified of versions) { + // Merge: start with previous expanded, apply the minified diff + const next = { ...current } + for (const [key, value] of Object.entries(minified)) { + if (value === '__unset') { + delete next[key] + } else { + next[key] = value + } + } + + // `next` is never mutated after this — the following iteration spreads it + // into a fresh object — so it can be pushed directly. + result.push(next as PackagistExpandedVersion) + current = next + } + + return result +} diff --git a/services/apps/packages_worker/src/packagist/fetchPackage.ts b/services/apps/packages_worker/src/packagist/fetchPackage.ts new file mode 100644 index 0000000000..28b4d3326e --- /dev/null +++ b/services/apps/packages_worker/src/packagist/fetchPackage.ts @@ -0,0 +1,227 @@ +import { Agent, type Dispatcher } from 'undici' + +import type { + FetchError, + P2Metadata, + P2NotModified, + PackagistMinifiedVersion, + PackagistStatsJson, +} from './types' + +// Shared dispatcher for all packagist fetches. autoSelectFamily races IPv4/IPv6 +// (Happy Eyeballs) — without it, an environment with broken IPv6 (common in +// containers) hangs on the first resolved AAAA address until UND_ERR_CONNECT_TIMEOUT. +// Connect timeout stays modest; the per-request 30s abort still bounds the whole call. +// `connections` caps sockets per origin at the crawl concurrency, so bursts queue on +// warm keep-alive connections instead of opening fresh ones — connection setup is +// where the intermittent UND_ERR_CONNECT_TIMEOUTs were observed under load. +export const packagistDispatcher: Dispatcher = new Agent({ + connect: { timeout: 15_000, autoSelectFamily: true }, + connections: 10, +}) + +export function buildPackagistUserAgent(): string { + const mailto = process.env.CROWD_PACKAGES_PACKAGIST_MAILTO || 'oss-packages@linuxfoundation.org' + return `lfx-packages-worker/0.1 (+https://lfx.linuxfoundation.org; mailto=${mailto})` +} + +// undici wraps network failures as `TypeError: fetch failed` with the real reason +// (ECONNRESET, ETIMEDOUT, DNS, …) on `cause` — surface it or the logs are undiagnosable. +export function describeFetchFailure(err: unknown): string { + const cause = (err as { cause?: { message?: string; code?: string } })?.cause + const detail = cause?.code ?? cause?.message + return detail ? `${String(err)} (cause: ${detail})` : String(err) +} + +// An error-path response whose body is never read can pin its socket instead of +// returning it to packagistDispatcher's pool (capped at 10 connections per origin) — +// canceling it releases the connection for reuse. Best-effort: a failure here must +// never mask the real error already being returned. Exported for listPackages.ts, +// which shares the same dispatcher and error-path risk. +export async function discardBody(res: Response): Promise { + try { + await res.body?.cancel() + } catch { + // ignore + } +} + +export async function fetchPackagistStats(name: string): Promise { + const url = `https://packagist.org/packages/${name}.json` + const abort = new AbortController() + const timer = setTimeout(() => abort.abort(), 30_000) + + try { + let res: Response + try { + // `dispatcher` is an undici-specific fetch option not present in the DOM RequestInit type. + const init: RequestInit & { dispatcher?: Dispatcher } = { + headers: { + Accept: 'application/json', + 'User-Agent': buildPackagistUserAgent(), + }, + signal: abort.signal, + dispatcher: packagistDispatcher, + } + res = await fetch(url, init as RequestInit) + } catch (err) { + return { kind: 'TRANSIENT', message: describeFetchFailure(err) } + } + + if (res.status === 404) { + await discardBody(res) + return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } + } + if (res.status === 429) { + await discardBody(res) + return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } + } + if (!res.ok) { + await discardBody(res) + return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + } + + let json: unknown + try { + json = await res.json() + } catch { + if (abort.signal.aborted) return { kind: 'TRANSIENT', message: 'body read timed out' } + return { kind: 'MALFORMED', message: 'invalid JSON' } + } + + // Shape guard: body.package must be an object with a string name + if (!isPackagistStatsJson(json)) { + return { kind: 'MALFORMED', message: 'unexpected shape' } + } + + return json + } finally { + clearTimeout(timer) + } +} + +export async function fetchPackagistP2( + name: string, + ifModifiedSince: string | null, +): Promise { + const url = `https://repo.packagist.org/p2/${name}.json` + const abort = new AbortController() + const timer = setTimeout(() => abort.abort(), 30_000) + + try { + const headers: Record = { + Accept: 'application/json', + 'User-Agent': buildPackagistUserAgent(), + } + if (ifModifiedSince !== null) { + headers['If-Modified-Since'] = ifModifiedSince + } + + let res: Response + try { + const init: RequestInit & { dispatcher?: Dispatcher } = { + headers, + signal: abort.signal, + dispatcher: packagistDispatcher, + } + res = await fetch(url, init as RequestInit) + } catch (err) { + return { kind: 'TRANSIENT', message: describeFetchFailure(err) } + } + + // 304 must be checked first + if (res.status === 304) { + return { kind: 'NOT_MODIFIED' } + } + + if (res.status === 404) { + await discardBody(res) + return { kind: 'NOT_FOUND', message: `${name} not found`, statusCode: 404 } + } + if (res.status === 429) { + await discardBody(res) + return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } + } + if (!res.ok) { + await discardBody(res) + return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + } + + let json: unknown + try { + json = await res.json() + } catch { + if (abort.signal.aborted) return { kind: 'TRANSIENT', message: 'body read timed out' } + return { kind: 'MALFORMED', message: 'invalid JSON' } + } + + // Shape guard: body.packages[name] must be an array + if (!isP2Response(json, name)) { + return { kind: 'MALFORMED', message: 'unexpected shape' } + } + + const minifiedVersions = json.packages[name] as PackagistMinifiedVersion[] + const lastModified = res.headers.get('last-modified') ?? null + + return { + minifiedVersions, + lastModified, + } + } finally { + clearTimeout(timer) + } +} + +// Validates every field normalizePackagistStats consumes unconditionally +// (blankToNull's .trim(), the maintainers .filter()) — a wrong runtime type here must +// surface as MALFORMED, not throw past the fast-retry/give-up path and get treated as +// a transient failure that Temporal retries forever on the same deterministic input. +function isPackagistStatsJson(v: unknown): v is PackagistStatsJson { + if (typeof v !== 'object' || v === null || !('package' in v)) return false + const pkg = (v as { package: unknown }).package + if (typeof pkg !== 'object' || pkg === null) return false + const p = pkg as Record + if (typeof p.name !== 'string') return false + if (p.description != null && typeof p.description !== 'string') return false + if (p.repository != null && typeof p.repository !== 'string') return false + if (p.maintainers != null && !Array.isArray(p.maintainers)) return false + return true +} + +// '__unset' is the p2 minified-diff sentinel for "field removed since the previous +// entry" — a legitimate value for any optional field, not just an absent key. +function isValidVersionField(value: unknown, isValidType: (v: unknown) => boolean): boolean { + return value === undefined || value === '__unset' || isValidType(value) +} + +// Validates every field normalize.ts consumes unconditionally on an expanded version: +// `version`/`version_normalized` via .startsWith()/.split()/.endsWith() +// (isPackagistDevVersion/isPackagistPrerelease), `homepage` via blankToNull's .trim(), +// `license` as the text[] written to versions.licenses, `time` as the timestamptz +// written to versions.published_at — checking it's merely a string isn't enough, since +// a non-date string like "not-a-date" would still be typeof 'string' but throw at the +// v.pub::timestamptz SQL cast, deep past this guard. `require`/`require-dev` are already +// guarded at their own call site (extractVersionDependencies checks typeof before +// Object.entries) so aren't re-validated here. +function isValidMinifiedVersion(v: unknown): boolean { + if (typeof v !== 'object' || v === null) return false + const entry = v as Record + return ( + typeof entry.version === 'string' && + isValidVersionField(entry.version_normalized, (x) => typeof x === 'string') && + isValidVersionField(entry.homepage, (x) => typeof x === 'string') && + isValidVersionField(entry.time, (x) => typeof x === 'string' && !isNaN(Date.parse(x))) && + isValidVersionField( + entry.license, + (x) => Array.isArray(x) && x.every((l) => typeof l === 'string'), + ) + ) +} + +function isP2Response(v: unknown, name: string): v is { packages: Record } { + if (typeof v !== 'object' || v === null || !('packages' in v)) return false + const packages = (v as { packages: unknown }).packages + if (typeof packages !== 'object' || packages === null) return false + const entries = (packages as Record)[name] + return Array.isArray(entries) && entries.every(isValidMinifiedVersion) +} diff --git a/services/apps/packages_worker/src/packagist/listPackages.ts b/services/apps/packages_worker/src/packagist/listPackages.ts new file mode 100644 index 0000000000..e906d988a6 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/listPackages.ts @@ -0,0 +1,129 @@ +import type { Dispatcher } from 'undici' + +import { + buildPackagistUserAgent, + describeFetchFailure, + discardBody, + packagistDispatcher, +} from './fetchPackage' +import type { FetchError } from './types' + +const PACKAGIST_LIST = 'https://packagist.org/packages/list.json' +// Non-backtracking form: a mandatory separator per repetition (vs. an optional one) +// removes the ambiguity that let the old `([_.-]?[a-z0-9]+)*` pattern backtrack +// exponentially on a long run of the same character class (CodeQL js/redos). +// Vendor and project segments have different rules per Composer's own name spec: the +// vendor allows only a single separator between alnum runs, but the project segment +// additionally allows up to two consecutive hyphens (e.g. `vendor/my--package` is a +// valid, real Composer name) — verified against Composer's ArrayLoader name pattern. +const COMPOSER_VENDOR_REGEX = /^[a-z0-9]+(?:[_.-][a-z0-9]+)*$/ +const COMPOSER_PROJECT_REGEX = /^[a-z0-9]+(?:(?:[_.]|-{1,2})[a-z0-9]+)*$/ + +export interface PackagistListEntry { + vendor: string + name: string + purl: string +} + +export function parsePackagistPackageList(json: unknown): { + entries: PackagistListEntry[] + invalid: number +} { + if (typeof json !== 'object' || json === null) { + throw new TypeError('list.json must be an object') + } + + const root = json as { packageNames?: unknown } + if (!Array.isArray(root.packageNames)) { + throw new TypeError('packageNames must be an array') + } + + const seen = new Set() + const entries: PackagistListEntry[] = [] + let invalid = 0 + + for (const item of root.packageNames) { + if (typeof item !== 'string') { + invalid++ + continue + } + + const lowercased = item.toLowerCase() + const parts = lowercased.split('/') + + // Validate: exactly one slash, each side non-empty and matches its composer name pattern + if ( + parts.length !== 2 || + !parts[0] || + !parts[1] || + !COMPOSER_VENDOR_REGEX.test(parts[0]) || + !COMPOSER_PROJECT_REGEX.test(parts[1]) + ) { + invalid++ + continue + } + + // Dedup on the lowercased form + if (seen.has(lowercased)) { + continue + } + seen.add(lowercased) + + entries.push({ + vendor: parts[0], + name: parts[1], + purl: `pkg:composer/${parts[0]}/${parts[1]}`, + }) + } + + return { entries, invalid } +} + +export async function fetchPackagistPackageList(): Promise { + const abort = new AbortController() + // 30s timer covering the body read too + const timer = setTimeout(() => abort.abort(), 30_000) + + try { + let res: Response + try { + const init: RequestInit & { dispatcher?: Dispatcher } = { + headers: { + Accept: 'application/json', + 'User-Agent': buildPackagistUserAgent(), + }, + signal: abort.signal, + dispatcher: packagistDispatcher, + } + res = await fetch(PACKAGIST_LIST, init as RequestInit) + } catch (err) { + return { kind: 'TRANSIENT', message: describeFetchFailure(err) } + } + + // Status classification: 404 NOT_FOUND, 429 RATE_LIMIT, other non-ok TRANSIENT + if (res.status === 404) { + await discardBody(res) + return { kind: 'NOT_FOUND', message: 'list not found', statusCode: 404 } + } + if (res.status === 429) { + await discardBody(res) + return { kind: 'RATE_LIMIT', message: 'rate limited', statusCode: 429 } + } + if (!res.ok) { + await discardBody(res) + return { kind: 'TRANSIENT', message: `HTTP ${res.status}`, statusCode: res.status } + } + + let json: unknown + try { + json = await res.json() + } catch { + if (abort.signal.aborted) return { kind: 'TRANSIENT', message: 'body read timed out' } + return { kind: 'MALFORMED', message: 'invalid JSON' } + } + + return json + } finally { + clearTimeout(timer) + } +} diff --git a/services/apps/packages_worker/src/packagist/normalize.ts b/services/apps/packages_worker/src/packagist/normalize.ts new file mode 100644 index 0000000000..9f4fc832ba --- /dev/null +++ b/services/apps/packages_worker/src/packagist/normalize.ts @@ -0,0 +1,225 @@ +import type { + NormalizedPackagistStats, + PackagistDependency, + PackagistExpandedVersion, + PackagistPackageInfo, + PackagistVersionRow, +} from './types' + +const PURL_COMPOSER_PREFIX = 'pkg:composer/' + +export function packagistNameFromPurl(purl: string): string { + return purl.slice(PURL_COMPOSER_PREFIX.length) +} + +function blankToNull(s: string | null | undefined): string | null { + if (s == null) return null + const t = s.trim() + return t || null +} + +// versions.licenses is documented as a deterministically-sorted SPDX array (schema +// comment, V1779710880__initial_schema.sql) — Composer allows dual/multi-licensed +// releases, so unlike npm/pypi's near-always-single-element input, the same license +// set can arrive from the registry in a different order between fetches. Sorting once +// here keeps semantically identical arrays byte-identical, avoiding false audit noise. +function sortLicenses(licenses: string[] | null | undefined): string[] | null { + return licenses && licenses.length > 0 ? [...licenses].sort() : null +} + +export function normalizePackagistStats(pkg: PackagistPackageInfo): NormalizedPackagistStats { + const description = blankToNull(pkg.description) + const repositoryUrl = blankToNull(pkg.repository) + + // Determine status: abandoned → deprecated, else active + let status: 'active' | 'deprecated' = 'active' + if (pkg.abandoned === true || (typeof pkg.abandoned === 'string' && pkg.abandoned)) { + status = 'deprecated' + } + + // Extract downloads, defaulting to null if not a number. downloads.monthly is + // deliberately not extracted here — packages.downloads_last_30d belongs exclusively + // to the dedicated downloads-30d lane's boundary-anchored snapshot. + const downloadsTotal = typeof pkg.downloads?.total === 'number' ? pkg.downloads.total : null + + const dependents = typeof pkg.dependents === 'number' ? pkg.dependents : null + + // Extract maintainers with non-empty names. isPackagistStatsJson only guarantees + // maintainers is an array — a rogue null/non-object element would throw on `m.name`. + const maintainers = (pkg.maintainers ?? []) + .filter( + (m): m is { name: string } => + typeof m === 'object' && m !== null && typeof m.name === 'string' && !!m.name, + ) + .map((m) => ({ + username: m.name, + displayName: null, + email: null, + role: 'maintainer' as const, + })) + + return { + name: pkg.name, + description, + repositoryUrl, + status, + dependents, + downloadsTotal, + maintainers, + } +} + +export function isPackagistDevVersion(version: string, versionNormalized?: string): boolean { + if (version.startsWith('dev-')) return true + if (versionNormalized?.endsWith('-dev')) return true + return false +} + +export function isPackagistPrerelease(versionNormalized: string): boolean { + const parts = versionNormalized.split('-') + if (parts.length < 2) return false + + const suffix = parts[1].toLowerCase() + + // Match: alpha|a|beta|b|rc followed by optional digits + // Stable: plain number, -patch\d*, -p\d* + if (suffix.match(/^(alpha|a|beta|b|rc)\d*$/)) { + return true + } + + return false +} + +// Custom compare — the repo's semver/osv versionCompare can't be reused here. +// Composer's version_normalized is 4-part (e.g. 2.0.0.0); semver.parse returns null +// on it, and the osv wrapper deliberately refuses lossy coercion. +const PRERELEASE_RANKS: Record = { alpha: 1, a: 1, beta: 2, b: 2, rc: 3 } +const STABLE_RANK = 4 + +function parseComposerVersion(v: string): { base: number[]; rank: number; suffixNum: number } { + const dashIdx = v.indexOf('-') + const base = (dashIdx === -1 ? v : v.slice(0, dashIdx)).split('.').map((p) => { + const n = Number(p) + return Number.isFinite(n) ? n : 0 + }) + + const suffixMatch = + dashIdx === -1 ? null : v.slice(dashIdx + 1).match(/^(alpha|a|beta|b|rc)(\d*)/i) + if (!suffixMatch) return { base, rank: STABLE_RANK, suffixNum: 0 } + + return { + base, + rank: PRERELEASE_RANKS[suffixMatch[1].toLowerCase()], + suffixNum: suffixMatch[2] ? Number(suffixMatch[2]) : 0, + } +} + +function compareComposerVersions(a: string, b: string): number { + const pa = parseComposerVersion(a) + const pb = parseComposerVersion(b) + + const maxLen = Math.max(pa.base.length, pb.base.length) + for (let i = 0; i < maxLen; i++) { + const diff = (pa.base[i] ?? 0) - (pb.base[i] ?? 0) + if (diff !== 0) return diff + } + + if (pa.rank !== pb.rank) return pa.rank - pb.rank + return pa.suffixNum - pb.suffixNum +} + +export function buildPackagistVersionRows(versions: PackagistExpandedVersion[]): { + versionRows: PackagistVersionRow[] + latestVersion: string | null + firstReleaseAt: string | null + latestReleaseAt: string | null + licenses: string[] | null + homepage: string | null +} { + // Filter out dev versions + const kept = versions.filter((v) => !isPackagistDevVersion(v.version, v.version_normalized)) + + if (kept.length === 0) { + return { + versionRows: [], + latestVersion: null, + firstReleaseAt: null, + latestReleaseAt: null, + licenses: null, + homepage: null, + } + } + + const versionRows: PackagistVersionRow[] = kept.map((v) => ({ + number: v.version, + publishedAt: v.time ?? null, + isLatest: false, // Will be set below + isPrerelease: isPackagistPrerelease(v.version_normalized ?? v.version), + licenses: sortLicenses(v.license), + })) + + // Find latest: prefer stable over prerelease; within each group use Composer ordering + let latestIdx = 0 + + for (let i = 1; i < kept.length; i++) { + if (versionRows[i].isPrerelease !== versionRows[latestIdx].isPrerelease) { + // A stable release always beats a prerelease + if (!versionRows[i].isPrerelease) latestIdx = i + continue + } + + const currentNorm = kept[i].version_normalized ?? kept[i].version + const latestNorm = kept[latestIdx].version_normalized ?? kept[latestIdx].version + if (compareComposerVersions(currentNorm, latestNorm) > 0) latestIdx = i + } + + versionRows[latestIdx].isLatest = true + + // Compute aggregates + const times = kept + .map((v) => v.time) + .filter((t): t is string => t != null) + .sort() + + const firstReleaseAt = times[0] ?? null + const latestReleaseAt = times[times.length - 1] ?? null + + // Licenses and homepage come from the latest row + const licenses = sortLicenses(kept[latestIdx].license) + const homepage = blankToNull(kept[latestIdx].homepage) + + return { + versionRows, + latestVersion: kept[latestIdx].version, + firstReleaseAt, + latestReleaseAt, + licenses, + homepage, + } +} + +export function extractVersionDependencies( + version: PackagistExpandedVersion, +): PackagistDependency[] { + const deps: PackagistDependency[] = [] + + // Add require (direct) dependencies, excluding platform targets + if (version.require && typeof version.require === 'object') { + for (const [name, constraint] of Object.entries(version.require)) { + if (typeof constraint === 'string' && name.includes('/')) { + deps.push({ name, constraint, kind: 'direct' }) + } + } + } + + // Add require-dev (dev) dependencies, excluding platform targets + if (version['require-dev'] && typeof version['require-dev'] === 'object') { + for (const [name, constraint] of Object.entries(version['require-dev'])) { + if (typeof constraint === 'string' && name.includes('/')) { + deps.push({ name, constraint, kind: 'dev' }) + } + } + } + + return deps +} diff --git a/services/apps/packages_worker/src/packagist/retryPolicy.ts b/services/apps/packages_worker/src/packagist/retryPolicy.ts new file mode 100644 index 0000000000..08ecd8fd56 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/retryPolicy.ts @@ -0,0 +1,4 @@ +// Per-package fetch attempts. Shared between the workflow's Temporal retry policy and the +// activity's give-up threshold so the two never drift: a package is only given up on +// (marked scanned-error so the cursor can advance) once Temporal has exhausted these attempts. +export const INGEST_MAX_ATTEMPTS = 5 diff --git a/services/apps/packages_worker/src/packagist/schedule.ts b/services/apps/packages_worker/src/packagist/schedule.ts new file mode 100644 index 0000000000..f81ba421cf --- /dev/null +++ b/services/apps/packages_worker/src/packagist/schedule.ts @@ -0,0 +1,79 @@ +import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client' + +// Cron minutes deliberately off :00 per Packagist crawler guidelines. +// The weekly metadata drain has no cron: seedPackagistPackages starts it as a +// child workflow once the seed completes, so sequencing is an event, not a +// clock offset. +export const PACKAGIST_CRONS = { + seed: '17 2 * * 0', + downloads30d: '53 3 1 * *', + // Late in the UTC day on purpose: Packagist's `daily` figure is mostly real data by + // 22:23 (vs. mostly borrowed from yesterday earlier on), with buffer before midnight. + downloadsDaily: '23 22 * * *', +} + +// Workflow types by name (not function reference) so this module doesn't pull the +// whole workflows index into consumers that only need the crons. +const SCHEDULES = [ + { + scheduleId: 'packagist-seed', + cron: PACKAGIST_CRONS.seed, + workflowType: 'seedPackagistPackages', + args: [] as unknown[], + }, + { + scheduleId: 'packagist-downloads-30d', + cron: PACKAGIST_CRONS.downloads30d, + workflowType: 'ingestPackagistDownloads30d', + args: [{}] as unknown[], + }, + { + scheduleId: 'packagist-downloads-daily', + cron: PACKAGIST_CRONS.downloadsDaily, + workflowType: 'ingestPackagistDownloadsDaily', + args: [{}] as unknown[], + }, +] + +export async function schedulePackagistIngest(): Promise { + // svc is imported lazily: its module graph can't load under vitest, and the wiring + // test imports PACKAGIST_CRONS from this file. The .js extension is required by + // node16 module resolution for dynamic imports. + const { svc } = await import('../service.js') + + const { temporal } = svc + if (!temporal) throw new Error('Temporal client not initialized') + + for (const schedule of SCHEDULES) { + try { + await temporal.schedule.create({ + scheduleId: schedule.scheduleId, + spec: { + cronExpressions: [schedule.cron], + }, + policies: { + overlap: ScheduleOverlapPolicy.SKIP, + catchupWindow: '1 hour', + }, + action: { + type: 'startWorkflow', + workflowType: schedule.workflowType, + taskQueue: 'packagist-worker', + workflowRunTimeout: '24 hours', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumAttempts: 3, + }, + args: schedule.args, + }, + }) + } catch (err) { + if (err instanceof ScheduleAlreadyRunning) { + svc.log.info(`Schedule ${schedule.scheduleId} already registered.`) + } else { + throw err + } + } + } +} diff --git a/services/apps/packages_worker/src/packagist/types.ts b/services/apps/packages_worker/src/packagist/types.ts new file mode 100644 index 0000000000..180a3bbb67 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/types.ts @@ -0,0 +1,114 @@ +import type { PackageMaintainerInput } from '@crowd/data-access-layer/src/packages/maintainers' + +export type FetchErrorKind = 'RATE_LIMIT' | 'TRANSIENT' | 'NOT_FOUND' | 'MALFORMED' + +export interface FetchError { + kind: FetchErrorKind + message: string + statusCode?: number +} + +const FETCH_ERROR_KINDS: ReadonlySet = new Set([ + 'RATE_LIMIT', + 'TRANSIENT', + 'NOT_FOUND', + 'MALFORMED', +]) + +// Explicit kind check (unlike pypi's loose `'kind' in v`) because the p2 fetch has a +// non-error NOT_MODIFIED outcome that also carries a `kind`. +export function isFetchError(v: unknown): v is FetchError { + return ( + typeof v === 'object' && + v !== null && + 'kind' in v && + FETCH_ERROR_KINDS.has((v as { kind: string }).kind) + ) +} + +// Subset of packagist.org/packages/{vendor}/{name}.json (dynamic stats endpoint) we consume. +export interface PackagistMaintainer { + name?: string + avatar_url?: string +} + +export interface PackagistDownloads { + total?: number + monthly?: number + daily?: number +} + +export interface PackagistPackageInfo { + name: string + description?: string | null + time?: string | null + maintainers?: PackagistMaintainer[] + type?: string | null + repository?: string | null + language?: string | null + // Absent/false when maintained; true or a replacement package name when abandoned. + abandoned?: boolean | string + dependents?: number + suggesters?: number + favers?: number + downloads?: PackagistDownloads +} + +export interface PackagistStatsJson { + package: PackagistPackageInfo +} + +export interface NormalizedPackagistStats { + // vendor/name, as Packagist reports it + name: string + description: string | null + repositoryUrl: string | null + status: 'active' | 'deprecated' + dependents: number | null + downloadsTotal: number | null + maintainers: PackageMaintainerInput[] +} + +// repo.packagist.org/p2/{vendor}/{name}.json — Composer-minified version objects. +// The first object is complete; each subsequent one carries only changed keys, with +// the literal string '__unset' marking a removed key. +export type PackagistMinifiedVersion = { version: string } & Record + +export interface PackagistExpandedVersion { + version: string + version_normalized?: string + time?: string + license?: string[] + homepage?: string + require?: Record + 'require-dev'?: Record + [key: string]: unknown +} + +export interface P2Metadata { + minifiedVersions: PackagistMinifiedVersion[] + // Last-Modified response header, replayed as If-Modified-Since on the next fetch. + lastModified: string | null +} + +export interface P2NotModified { + kind: 'NOT_MODIFIED' +} + +export function isP2NotModified(v: unknown): v is P2NotModified { + return typeof v === 'object' && v !== null && (v as { kind?: string }).kind === 'NOT_MODIFIED' +} + +export interface PackagistVersionRow { + number: string + publishedAt: string | null + isLatest: boolean + isPrerelease: boolean + licenses: string[] | null +} + +export interface PackagistDependency { + name: string + constraint: string + kind: 'direct' | 'dev' +} diff --git a/services/apps/packages_worker/src/packagist/upsertMetadata.ts b/services/apps/packages_worker/src/packagist/upsertMetadata.ts new file mode 100644 index 0000000000..bdd594c8f9 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/upsertMetadata.ts @@ -0,0 +1,118 @@ +import { + getPackagistPackageIdsByNames, + logAuditFieldChanges, + reconcileVersionDependencies, + updatePackagistVersionAggregates, + upsertPackagistVersions, +} from '@crowd/data-access-layer/src/packages' +import type { VersionDependencyEdge } from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { stripNullBytesDeep } from '../utils/stripNullBytesDeep' + +import { + buildPackagistVersionRows, + extractVersionDependencies, + isPackagistDevVersion, +} from './normalize' +import type { PackagistDependency, PackagistExpandedVersion } from './types' + +const WORKER = 'packagist' + +export async function persistPackagistMetadata( + qx: QueryExecutor, + purl: string, + expanded: PackagistExpandedVersion[], +): Promise<{ found: boolean; changedFields: string[]; unresolvedDependencyTargets: number }> { + // Registry data can contain NUL bytes (e.g. mojibake descriptions/licenses) that + // Postgres text columns reject; strip them before any field is persisted. + stripNullBytesDeep(expanded) + + const { versionRows, latestVersion, firstReleaseAt, latestReleaseAt, licenses, homepage } = + buildPackagistVersionRows(expanded) + + let found = false + const changedFields: string[] = [] + let unresolvedDependencyTargets = 0 + + // One transaction for the whole p2 write path AND its audit record: the aggregates + // update, version upsert + is_latest cleanup, and dependency reconcile (delete stale + // + upsert current) are each multi-statement on their own — sharing one tx means a + // failure partway through (including a failed audit insert) can never leave + // versions/dependencies half-refreshed or their audit trail out of sync. + await qx.tx(async (t) => { + const agg = await updatePackagistVersionAggregates(t, purl, { + versionsCount: versionRows.length, + latestVersion, + firstReleaseAt, + latestReleaseAt, + licenses, + homepage, + }) + + if (!agg) return + + found = true + changedFields.push(...agg.changedFields) + + let versionIds: Array<{ number: string; id: string }> = [] + if (versionRows.length > 0) { + const versionResult = await upsertPackagistVersions(t, agg.id, versionRows, latestVersion) + changedFields.push(...versionResult.changedFields) + versionIds = versionResult.versionIds + } + + // One pass over the tagged versions collects target names and provisional edges; + // target ids are then resolved in a single batch query. + const versionMap = new Map(versionIds.map((v) => [v.number, v.id])) + const targetNames = new Set() + const pending: Array<{ versionId: string; dep: PackagistDependency }> = [] + + for (const v of expanded) { + if (isPackagistDevVersion(v.version, v.version_normalized)) continue + const versionId = versionMap.get(v.version) + if (!versionId) continue + + for (const dep of extractVersionDependencies(v)) { + targetNames.add(dep.name) + pending.push({ versionId, dep }) + } + } + + // Reconciled for every version being refreshed (not just ones with a resolved + // dependency this pass) — a version whose manifest drops a requirement, or all of + // them, must have its stale package_dependencies rows removed, not just the newly + // declared ones upserted. + if (versionIds.length > 0) { + const edges: VersionDependencyEdge[] = [] + if (pending.length > 0) { + const idMap = await getPackagistPackageIdsByNames(t, Array.from(targetNames)) + for (const { versionId, dep } of pending) { + const dependsOnId = idMap.get(dep.name) + if (!dependsOnId) { + unresolvedDependencyTargets++ + continue + } + edges.push({ + packageId: agg.id, + versionId, + dependsOnId, + constraint: dep.constraint, + kind: dep.kind, + }) + } + } + + const depChanges = await reconcileVersionDependencies( + t, + versionIds.map((v) => v.id), + edges, + ) + changedFields.push(...depChanges) + } + + await logAuditFieldChanges(t, WORKER, purl, changedFields) + }) + + return { found, changedFields, unresolvedDependencyTargets } +} diff --git a/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts new file mode 100644 index 0000000000..a975f4cb56 --- /dev/null +++ b/services/apps/packages_worker/src/packagist/upsertPackageInfo.ts @@ -0,0 +1,101 @@ +import { + getOrCreateRepoByUrl, + logAuditFieldChanges, + removeDeclaredPackageRepo, + updatePackagistPackageStats, + upsertPackageMaintainers, + upsertPackageRepoPreserveProvenance, +} from '@crowd/data-access-layer/src/packages' +import type { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { canonicalizeRepoUrl } from '../utils/canonicalizeRepoUrl' +import { stripNullBytesDeep } from '../utils/stripNullBytesDeep' + +import type { NormalizedPackagistStats } from './types' + +const WORKER = 'packagist' + +// Dynamic-endpoint persistence: packages fields + repo link for ALL packages, +// maintainers only for critical ones. Download rows are NOT written here — they +// belong to the dedicated downloads-30d/daily lanes. All writes AND the audit +// record share one transaction so a failure partway through — including a +// failed audit insert — can never leave the writes and their audit trail +// inconsistent with each other, and a retry can't lose an already-committed +// change's audit event. +export async function persistPackagistPackageInfo( + qx: QueryExecutor, + purl: string, + stats: NormalizedPackagistStats, +): Promise<{ found: boolean; changedFields: string[] }> { + // Registry data can contain NUL bytes (e.g. mojibake descriptions) that Postgres + // text columns reject; strip them before any field is persisted. + stripNullBytesDeep(stats) + + const canonical = stats.repositoryUrl ? canonicalizeRepoUrl(stats.repositoryUrl) : null + // Packagist's repository field is free-form/author-supplied. canonicalizeRepoUrl's + // 'other' bucket also matches non-repo URLs (wikis, issue trackers, registry pages) + // that happen to have 2+ path segments, so only trust the verified SCM hosts here — + // github.com/gitlab.com/bitbucket.org — rather than the shared utility's default, + // which other callers (npm/maven/cargo) rely on staying permissive. + const trustedRepo = canonical && canonical.host !== 'other' ? canonical : null + + let found = false + const changedFields: string[] = [] + + await qx.tx(async (t) => { + // Step 1: Update packages row + const result = await updatePackagistPackageStats(t, { + purl, + description: stats.description, + declaredRepositoryUrl: stats.repositoryUrl, + repositoryUrl: trustedRepo?.url ?? null, + status: stats.status, + totalDownloads: stats.downloadsTotal, + dependentCount: stats.dependents, + }) + + if (!result) return + + found = true + const { id, isCritical } = result + changedFields.push(...result.changedFields) + + // Step 2: Link the repo for ALL packages — 'declared'/0.8 is the manifest-declared + // convention shared by npm/pypi/maven/cargo. Preserve-provenance: a higher-confidence + // link another pipeline (manual/deps_dev) already owns for this repo must not be + // downgraded to 'declared'/0.8 by a routine weekly refresh — matches + // upsertMavenPackageRepo/cargo's established GREATEST-confidence, source-preserving + // pattern. When there's no trusted repo (removed from the manifest, or no longer + // canonicalizable to a known host), or it now resolves to a different repo, clear any + // previously-declared link that no longer applies — package_repos' unique key is + // (package_id, repo_id), not (package_id, source), so upserting the new link alone + // would leave a stale one dangling. + if (trustedRepo) { + const repo = await getOrCreateRepoByUrl(t, trustedRepo.url, trustedRepo.host) + const linkChanged = await upsertPackageRepoPreserveProvenance(t, id, repo.id, 'declared', 0.8) + const removedFields = await removeDeclaredPackageRepo(t, id, repo.id) + changedFields.push(...repo.changedFields, ...linkChanged, ...removedFields) + } else { + const removedFields = await removeDeclaredPackageRepo(t, id) + changedFields.push(...removedFields) + } + + // Step 3: Maintainers only for critical packages. upsertPackageMaintainers always + // replaces the full stored set (including deleting rows for maintainers no longer + // reported), so it must run even when the registry now reports zero maintainers — + // skipping it on an empty list would leave stale maintainers attached forever. + if (isCritical) { + const maintainerChanges = await upsertPackageMaintainers( + t, + id, + stats.maintainers, + 'packagist', + ) + changedFields.push(...maintainerChanges) + } + + await logAuditFieldChanges(t, WORKER, purl, changedFields) + }) + + return { found, changedFields } +} diff --git a/services/apps/packages_worker/src/packagist/workflows.ts b/services/apps/packages_worker/src/packagist/workflows.ts new file mode 100644 index 0000000000..52be4453eb --- /dev/null +++ b/services/apps/packages_worker/src/packagist/workflows.ts @@ -0,0 +1,133 @@ +import { + ParentClosePolicy, + WorkflowIdReusePolicy, + continueAsNew, + log, + proxyActivities, + startChild, +} from '@temporalio/workflow' + +import type * as activities from './activities' +import { INGEST_MAX_ATTEMPTS } from './retryPolicy' + +const acts = proxyActivities({ + startToCloseTimeout: '15 minutes', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumAttempts: INGEST_MAX_ATTEMPTS, + }, +}) + +const INGEST_BATCH = 50 +const ROUNDS_PER_RUN = 20 + +interface MetadataState { + cutoff?: string + cursor?: string +} + +interface DownloadsState { + cutoff?: string + cursor?: string +} + +export async function seedPackagistPackages(): Promise { + await acts.runPackagistPackageSeed() + + // Chain the drain off seed completion (not a cron) so newly discovered packages exist + // as rows first. ABANDON so it outlives this workflow; fixed id + ALLOW_DUPLICATE means + // a drain that outlasts the week makes next Sunday's seed skip its start instead of + // doubling the crawl (a still-RUNNING id always throws regardless of reuse policy). + try { + await startChild(ingestPackagistMetadata, { + workflowId: 'packagist-metadata-drain', + workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE, + args: [{}], + parentClosePolicy: ParentClosePolicy.ABANDON, + }) + } catch (err) { + if (err instanceof Error && err.name === 'WorkflowExecutionAlreadyStartedError') { + log.warn('packagist metadata drain still running from a prior seed — skipping chain-start') + return + } + throw err + } +} + +// The cutoff is fixed once per run (deterministic activity), same pattern as the +// downloads-30d/daily lanes — a keyset scan only ever visits each purl once per drain, +// so due-selection must be anchored to a stable point in time rather than a live NOW() +// that would let a purl processed early in the run dodge this cycle's refresh window. +export async function ingestPackagistMetadata(state: MetadataState = {}): Promise { + const cutoff = state.cutoff ?? (await acts.packagistCurrentTimestamp()) + let cursor = state.cursor || '' + const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() + + for (let r = 0; r < ROUNDS_PER_RUN; r++) { + const { candidates, nextCursor } = await acts.getPackagistMetadataBatch( + cutoff, + cursor, + INGEST_BATCH, + ) + if (candidates.length === 0) return + await acts.ingestPackagistMetadataBatch(candidates) + cursor = nextCursor + if (stopAfterFirstPage) return + if (candidates.length < INGEST_BATCH) return + } + + await continueAsNew({ cutoff, cursor }) +} + +// Monthly capture of the observed rolling 30d window for every packagist package. +// The cutoff is fixed once per run (deterministic activity) so the watermark-based +// due-selection drains the whole universe exactly once per cron fire. +export async function ingestPackagistDownloads30d(state: DownloadsState = {}): Promise { + const cutoff = state.cutoff ?? (await acts.packagistCurrentTimestamp()) + // Packagist's monthly window is labeled by calendar month (see downloads.ts), so the + // write-date must come from the run's fixed cutoff — a drain that runs past real UTC + // midnight on the 1st must not let later batches slide into the next month's window. + const runDate = cutoff.slice(0, 10) + let cursor = state.cursor || '' + const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() + + for (let r = 0; r < ROUNDS_PER_RUN; r++) { + const { purls, nextCursor } = await acts.getPackagist30dBatch(cutoff, cursor, INGEST_BATCH) + if (purls.length === 0) return + await acts.ingestPackagist30dBatch(purls, runDate) + cursor = nextCursor + if (stopAfterFirstPage) return + if (purls.length < INGEST_BATCH) return + } + + await continueAsNew({ cutoff, cursor }) +} + +// Daily downloads capture for the critical slice. +export async function ingestPackagistDownloadsDaily(state: DownloadsState = {}): Promise { + if ((await acts.getCriticalPackagistCount()) === 0) return + + const cutoff = state.cutoff ?? (await acts.packagistCurrentTimestamp()) + // Packagist's `daily` figure is tied to a specific calendar day (see schedule.ts) — + // derive the write-date from the run's fixed cutoff instead of re-reading the clock + // per batch, so a drain that runs past UTC midnight still tags every row consistently. + const runDate = cutoff.slice(0, 10) + let cursor = state.cursor || '' + const stopAfterFirstPage = await acts.packagistStopAfterFirstPage() + + for (let r = 0; r < ROUNDS_PER_RUN; r++) { + const { candidates, nextCursor } = await acts.getPackagistDailyBatch( + cutoff, + cursor, + INGEST_BATCH, + ) + if (candidates.length === 0) return + await acts.ingestPackagistDailyBatch(candidates, runDate) + cursor = nextCursor + if (stopAfterFirstPage) return + if (candidates.length < INGEST_BATCH) return + } + + await continueAsNew({ cutoff, cursor }) +} diff --git a/services/apps/packages_worker/src/scripts/triggerPackagistSeed.ts b/services/apps/packages_worker/src/scripts/triggerPackagistSeed.ts new file mode 100644 index 0000000000..48da3b31e8 --- /dev/null +++ b/services/apps/packages_worker/src/scripts/triggerPackagistSeed.ts @@ -0,0 +1,82 @@ +import { TEMPORAL_CONFIG, getTemporalClient } from '@crowd/temporal' + +import { + ingestPackagistDownloads30d, + ingestPackagistDownloadsDaily, + ingestPackagistMetadata, + seedPackagistPackages, +} from '../packagist/workflows' + +const HELP = ` +Usage: trigger-packagist [seed|metadata|downloads-30d|downloads-daily] + +Arguments: + seed Fetch packagist.org/packages/list.json and seed the packages table (default) + metadata Crawl the dynamic (package info) + p2 (versions/dependencies) endpoints + downloads-30d Capture the observed rolling 30d window for every package + downloads-daily Capture daily downloads for the critical slice + +Examples: + pnpm trigger-packagist:local + pnpm trigger-packagist:local seed + pnpm trigger-packagist:local metadata + pnpm trigger-packagist:local downloads-30d + pnpm trigger-packagist:local downloads-daily +` + +const TARGETS = ['seed', 'metadata', 'downloads-30d', 'downloads-daily'] as const +type Target = (typeof TARGETS)[number] + +async function main(): Promise { + const args = process.argv.slice(2) + if (args.includes('--help') || args.includes('-h')) { + console.log(HELP) + process.exit(0) + } + + const target = (args[0] ?? 'seed') as Target + if (!TARGETS.includes(target)) { + console.error(`Unknown target "${target}". Use one of: ${TARGETS.join(', ')}.`) + process.exit(1) + } + + const cfg = TEMPORAL_CONFIG() + if (!cfg.serverUrl || !cfg.namespace) { + console.error('Missing CROWD_TEMPORAL_SERVER_URL or CROWD_TEMPORAL_NAMESPACE') + process.exit(1) + } + + const client = await getTemporalClient(cfg) + const now = Date.now() + + if (target === 'seed') { + const handle = await client.workflow.start(seedPackagistPackages, { + taskQueue: 'packagist-worker', + workflowId: `packagist-seed-manual-${now}`, + args: [], + }) + console.log(`Started workflow ${handle.workflowId}`) + return + } + + const workflow = + target === 'metadata' + ? ingestPackagistMetadata + : target === 'downloads-30d' + ? ingestPackagistDownloads30d + : ingestPackagistDownloadsDaily + + const handle = await client.workflow.start(workflow, { + taskQueue: 'packagist-worker', + workflowId: `packagist-${target}-manual-${now}`, + args: [{}], + }) + console.log(`Started workflow ${handle.workflowId}`) +} + +main() + .then(() => process.exit(0)) + .catch((err) => { + console.error('Failed to trigger packagist workflow:', err) + process.exit(1) + }) diff --git a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts index dcd61f4c9f..e183aff207 100644 --- a/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts +++ b/services/apps/packages_worker/src/security-contacts/extractors/registry/index.ts @@ -17,15 +17,17 @@ type EcosystemFetcher = ( repoUrl?: string, ) => Promise -// Keyed by the lowercased packages.ecosystem value. -const FETCHERS: Record = { +// Keyed by the lowercased packages.ecosystem value. Composer packages live under +// 'packagist' — the value the packagist worker seeds — not 'composer', which no +// writer ever produces (the original entry predated the worker and guessed wrong). +export const FETCHERS: Record = { npm: fetchNpm, pypi: fetchPypi, maven: fetchMaven, cargo: fetchCargo, nuget: fetchNuget, rubygems: fetchRubygems, - composer: fetchComposer, + packagist: fetchComposer, go: fetchGo, } diff --git a/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts b/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts index 544c368b30..0388081239 100644 --- a/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts +++ b/services/apps/packages_worker/src/utils/__tests__/canonicalizeRepoUrl.test.ts @@ -29,6 +29,55 @@ describe('canonicalizeRepoUrl', () => { 'github', ], ['ssh://git@github.com:2222/foo/bar.git', 'https://github.com/foo/bar', 'github'], + [ + 'https://gitlab.com/group/subgroup/project', + 'https://gitlab.com/group/subgroup/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/subgroup/subsubgroup/project.git', + 'https://gitlab.com/group/subgroup/subsubgroup/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/project/-/tree/master/src', + 'https://gitlab.com/group/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/subgroup/project/-/blob/main/README.md', + 'https://gitlab.com/group/subgroup/project', + 'gitlab', + ], + [ + // Pre-2018 GitLab / shorthand copies: no `/-/` marker ahead of the deep-link. + 'https://gitlab.com/group/project/tree/master/src', + 'https://gitlab.com/group/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/subgroup/project/blob/main/README.md', + 'https://gitlab.com/group/subgroup/project', + 'gitlab', + ], + [ + // The reviewer's exact regression example: `raw` was missing from the legacy + // route list, so this previously resolved as the bogus nested repo + // `group/project/raw`. + 'https://gitlab.com/group/project/raw/main/file.php', + 'https://gitlab.com/group/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/project/blame/main/file.php', + 'https://gitlab.com/group/project', + 'gitlab', + ], + [ + 'https://gitlab.com/group/subgroup/project/issues/42', + 'https://gitlab.com/group/subgroup/project', + 'gitlab', + ], ])('canonicalizes %s', (input, expectedUrl, expectedHost) => { expect(canonicalizeRepoUrl(input)).toEqual({ url: expectedUrl, host: expectedHost }) }) diff --git a/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts b/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts index a308697a45..b08617c940 100644 --- a/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts +++ b/services/apps/packages_worker/src/utils/canonicalizeRepoUrl.ts @@ -24,6 +24,34 @@ const HOST_ENUM: Record = { // same repo never produces two distinct repos.url keys. const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) +// Pre-2018 GitLab URLs (and shorthand copies of them still circulating) mark deep-links +// the same way GitHub does — appended directly after the project path, with no `/-/` +// separator. GitLab reserves these route words at the project-slug position precisely so +// they can never collide with a real project name (docs: user/reserved_names, the +// PROJECT_WILDCARD_ROUTES list), so treating the first one as a deep-link boundary is +// safe even for arbitrarily nested subgroups. +const GITLAB_LEGACY_ROUTE_SEGMENTS = new Set([ + 'tree', + 'blob', + 'blame', + 'raw', + 'commits', + 'commit', + 'compare', + 'issues', + 'merge_requests', + 'wikis', + 'builds', + 'create', + 'create_dir', + 'edit', + 'find_file', + 'new', + 'preview', + 'refs', + 'update', +]) + /** * Canonicalize a source-repository URL to `{ url, host }` where url is * `https:////` and host is the coarse classification stored @@ -32,9 +60,13 @@ const CASE_INSENSITIVE_HOSTS = new Set(['github.com', 'gitlab.com']) * Shared across the registry sub-workers (npm, Maven, …) and the GitHub * enricher so `repos.url` keys never diverge per ADR 0001. Handles npm * shorthand (`github:owner/repo`, bare `owner/repo`), SSH scp form, `ssh://`, - * `git+`, `git://`, `www.`, and monorepo `/tree//` deep-links - * (only the first two path segments are kept). Returns null when the input - * cannot be reduced to an owner/name pair. + * `git+`, `git://`, `www.`, and monorepo deep-links: GitHub/Bitbucket's bare + * `/tree//` (only the first two path segments are kept) and + * GitLab's `/-/tree//` (kept segments run up to the `/-/`, + * preserving arbitrarily nested subgroups) — plus a legacy fallback for pre-2018 + * GitLab links with no `/-/` marker, cut at the first reserved route keyword + * (`tree`, `blob`, `issues`, ...) instead. Returns null when the input cannot be + * reduced to an owner/name pair. */ export function canonicalizeRepoUrl(raw: string): CanonicalRepo | null { let s = raw.trim().replace(/#.*$/, '') @@ -80,8 +112,28 @@ export function canonicalizeRepoUrl(raw: string): CanonicalRepo | null { if (segments.length < 2) return null const isKnownHost = hostname in HOST_ENUM - let ownerPath = isKnownHost ? [segments[0]] : segments.slice(0, -1) - let name = (isKnownHost ? segments[1] : segments[segments.length - 1]).replace(/\.git$/, '') + // GitLab uniquely supports arbitrarily nested subgroups (group/subgroup/.../project), + // unlike GitHub/Bitbucket's flat owner/repo. Its deep-link suffixes (tree, blob, issues, + // merge_requests, ...) are marked off by a `/-/` path segment rather than appended + // directly after the repo path, so split there instead of truncating to 2 segments. + let pathSegments: string[] + if (hostname === 'gitlab.com') { + const dashIdx = segments.indexOf('-') + // Legacy fallback: a route keyword with no `/-/` ahead of it (index 2+, so at least + // group + project survive) also marks the boundary — whichever comes first wins. + const legacyIdx = segments.findIndex( + (seg, i) => i >= 2 && GITLAB_LEGACY_ROUTE_SEGMENTS.has(seg), + ) + const cutIdx = [dashIdx, legacyIdx].filter((i) => i !== -1).sort((a, b) => a - b)[0] + pathSegments = cutIdx === undefined ? segments : segments.slice(0, cutIdx) + } else if (isKnownHost) { + pathSegments = segments.slice(0, 2) + } else { + pathSegments = segments + } + + let ownerPath = pathSegments.slice(0, -1) + let name = (pathSegments[pathSegments.length - 1] ?? '').replace(/\.git$/, '') if (!name || ownerPath.length === 0 || ownerPath.some((seg) => !seg)) return null if (CASE_INSENSITIVE_HOSTS.has(hostname)) { diff --git a/services/apps/packages_worker/src/workflows/index.ts b/services/apps/packages_worker/src/workflows/index.ts index 439566dfe4..4b9712187b 100644 --- a/services/apps/packages_worker/src/workflows/index.ts +++ b/services/apps/packages_worker/src/workflows/index.ts @@ -25,6 +25,12 @@ export { ingestPypiDownloadsDaily, } from '../pypi/downloads/ingestPypiDownloads' export { ingestNuGetPackages } from '../nuget/workflows' +export { + seedPackagistPackages, + ingestPackagistMetadata, + ingestPackagistDownloads30d, + ingestPackagistDownloadsDaily, +} from '../packagist/workflows' export { ingestRubyGemsCriticalDetails, ingestRubyGemsPackages } from '../rubygems/workflows' export { ingestSecurityContacts, diff --git a/services/apps/packages_worker/vitest.config.ts b/services/apps/packages_worker/vitest.config.ts index 530cc66032..d3c0b9dc1e 100644 --- a/services/apps/packages_worker/vitest.config.ts +++ b/services/apps/packages_worker/vitest.config.ts @@ -1,13 +1,43 @@ import { defineConfig } from 'vitest/config' +const shared = { + environment: 'node' as const, + server: { + deps: { + inline: [/@crowd\//], + }, + }, +} + export default defineConfig({ test: { - environment: 'node', - include: ['src/**/*.test.ts'], - server: { - deps: { - inline: [/@crowd\//], + projects: [ + { + test: { + ...shared, + name: 'packagist', + include: ['src/packagist/**/*.test.ts'], + // getPackagesDb() builds a lazy pg-promise pool from these at activity call + // time; the packagist suites mock all queries so nothing ever connects. + // Scoped to this project so DB-gated integration suites elsewhere (which + // auto-skip when the real vars are absent) are unaffected. + env: { + CROWD_PACKAGES_DB_WRITE_HOST: '127.0.0.1', + CROWD_PACKAGES_DB_PORT: '5432', + CROWD_PACKAGES_DB_DATABASE: 'packages-test', + CROWD_PACKAGES_DB_USERNAME: 'test', + CROWD_PACKAGES_DB_PASSWORD: 'test', + }, + }, }, - }, + { + test: { + ...shared, + name: 'default', + include: ['src/**/*.test.ts'], + exclude: ['src/packagist/**'], + }, + }, + ], }, }) diff --git a/services/libs/data-access-layer/src/packages/dependencies.ts b/services/libs/data-access-layer/src/packages/dependencies.ts new file mode 100644 index 0000000000..298ed1f043 --- /dev/null +++ b/services/libs/data-access-layer/src/packages/dependencies.ts @@ -0,0 +1,93 @@ +import { QueryExecutor } from '../queryExecutor' + +export interface VersionDependencyEdge { + packageId: string + versionId: string + dependsOnId: string + constraint: string + kind: 'direct' | 'dev' +} + +// Returns a batch-level changed-fields summary (like the sibling upsert* functions), +// not a per-edge diff: a batch can touch many edges across many versions in one call, +// so this reports whether ANY edge in the batch was newly inserted / had its +// constraint change, rather than one row per edge. +export async function upsertVersionDependencies( + qx: QueryExecutor, + edges: VersionDependencyEdge[], +): Promise { + if (edges.length === 0) return [] + + const packageIds = edges.map((e) => e.packageId) + const versionIds = edges.map((e) => e.versionId) + const dependsOnIds = edges.map((e) => e.dependsOnId) + const constraints = edges.map((e) => e.constraint) + const kinds = edges.map((e) => e.kind) + + const row: { changed_fields: string[] } = await qx.selectOne( + `WITH old AS ( + SELECT version_id, depends_on_id, dependency_kind, version_constraint + FROM package_dependencies + WHERE version_id = ANY($(versionIds)::bigint[]) + AND depends_on_id = ANY($(dependsOnIds)::bigint[]) + ), + ins AS ( + INSERT INTO package_dependencies (package_id, version_id, depends_on_id, version_constraint, dependency_kind, is_optional, created_at, updated_at) + SELECT e.package_id, e.version_id, e.depends_on_id, e.version_constraint, e.dependency_kind, FALSE, NOW(), NOW() + FROM unnest($(packageIds)::bigint[], $(versionIds)::bigint[], $(dependsOnIds)::bigint[], + $(constraints)::text[], $(kinds)::text[]) + AS e(package_id, version_id, depends_on_id, version_constraint, dependency_kind) + ON CONFLICT (version_id, depends_on_id, dependency_kind) DO UPDATE SET + version_constraint = EXCLUDED.version_constraint, + updated_at = NOW() + RETURNING version_id, depends_on_id, dependency_kind, version_constraint + ) + SELECT array_remove(ARRAY[ + CASE WHEN bool_or(o.version_id IS NULL) THEN 'package_dependencies.depends_on_id' END, + CASE WHEN bool_or(o.version_id IS NOT NULL AND o.version_constraint IS DISTINCT FROM ins.version_constraint) + THEN 'package_dependencies.version_constraint' END + ], NULL) AS changed_fields + FROM ins + LEFT JOIN old o + ON o.version_id = ins.version_id + AND o.depends_on_id = ins.depends_on_id + AND o.dependency_kind = ins.dependency_kind`, + { packageIds, versionIds, dependsOnIds, constraints, kinds }, + ) + return row.changed_fields +} + +// Reconciles ALL dependency edges for a set of versions being refreshed: deletes any +// existing package_dependencies row for those versions that isn't in `edges` (a +// requirement removed from the manifest, or a version that now declares none at all), +// then upserts the current edges. Scoped to `versionIds` — version_id has a supporting +// btree index, so this stays a bounded, per-package operation despite the +// depends_on_id HASH-partitioning. +export async function reconcileVersionDependencies( + qx: QueryExecutor, + versionIds: string[], + edges: VersionDependencyEdge[], +): Promise { + if (versionIds.length === 0) return [] + + const deletedCount = await qx.result( + `DELETE FROM package_dependencies + WHERE version_id = ANY($(versionIds)::bigint[]) + AND NOT EXISTS ( + SELECT 1 FROM unnest($(keepVersionIds)::bigint[], $(keepDependsOnIds)::bigint[], $(keepKinds)::text[]) + AS keep(version_id, depends_on_id, dependency_kind) + WHERE keep.version_id = package_dependencies.version_id + AND keep.depends_on_id = package_dependencies.depends_on_id + AND keep.dependency_kind = package_dependencies.dependency_kind + )`, + { + versionIds, + keepVersionIds: edges.map((e) => e.versionId), + keepDependsOnIds: edges.map((e) => e.dependsOnId), + keepKinds: edges.map((e) => e.kind), + }, + ) + + const upsertChanges = await upsertVersionDependencies(qx, edges) + return deletedCount > 0 ? [...upsertChanges, 'package_dependencies.depends_on_id'] : upsertChanges +} diff --git a/services/libs/data-access-layer/src/packages/downloadsLast30d.integration.test.ts b/services/libs/data-access-layer/src/packages/downloadsLast30d.integration.test.ts new file mode 100644 index 0000000000..ceac6c2190 --- /dev/null +++ b/services/libs/data-access-layer/src/packages/downloadsLast30d.integration.test.ts @@ -0,0 +1,146 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { getDbConnection } from '@crowd/database' + +import type { QueryExecutor } from '../queryExecutor' +import { pgpQx } from '../queryExecutor' + +import { insertLast30dDownloadIfAbsent } from './downloadsLast30d' + +// Integration test: hits the running packages-db. Skipped automatically when any of +// the DB env vars are missing so unit-test runs in CI stay green. +const HAVE_DB = + !!process.env.CROWD_PACKAGES_DB_WRITE_HOST && + !!process.env.CROWD_PACKAGES_DB_PORT && + !!process.env.CROWD_PACKAGES_DB_USERNAME && + !!process.env.CROWD_PACKAGES_DB_DATABASE && + !!process.env.CROWD_PACKAGES_DB_PASSWORD + +const FIXTURE_TAG = 'akrites-downloads-last-30d-fixture' + +// insertLast30dDownloadIfAbsent is packagist's "first observation wins" guard for the +// monthly rolling window: the presence check, insert, and packages mirror all happen +// in one statement (ON CONFLICT DO NOTHING) so a genuine race between two callers for +// the same purl+month can never let a later write silently overwrite an earlier one. +describe.skipIf(!HAVE_DB)('insertLast30dDownloadIfAbsent — real packages-db', () => { + let qx: QueryExecutor + + async function cleanupFixtures(): Promise { + await qx.result( + `DELETE FROM downloads_last_30d WHERE purl IN ( + SELECT purl FROM packages WHERE ingestion_source = $(tag))`, + { tag: FIXTURE_TAG }, + ) + await qx.result(`DELETE FROM packages WHERE ingestion_source = $(tag)`, { tag: FIXTURE_TAG }) + } + + async function makePackage(purl: string): Promise { + await qx.result( + `INSERT INTO packages (purl, ecosystem, namespace, name, registry_url, status, ingestion_source) + VALUES ($(purl), 'packagist', 'fixture', $(purl), 'https://example.test', 'active', $(tag))`, + { purl, tag: FIXTURE_TAG }, + ) + } + + beforeAll(async () => { + const conn = await getDbConnection({ + host: process.env.CROWD_PACKAGES_DB_WRITE_HOST ?? '', + port: parseInt(process.env.CROWD_PACKAGES_DB_PORT ?? '0', 10), + database: process.env.CROWD_PACKAGES_DB_DATABASE ?? '', + user: process.env.CROWD_PACKAGES_DB_USERNAME ?? '', + password: process.env.CROWD_PACKAGES_DB_PASSWORD ?? '', + }) + qx = pgpQx(conn) + await cleanupFixtures() + }, 30_000) + + afterAll(async () => { + if (qx) await cleanupFixtures() + }) + + it('inserts the window and mirrors it to packages on the first call', async () => { + const purl = `pkg:composer/${FIXTURE_TAG}/first-call` + await makePackage(purl) + + const changed = await insertLast30dDownloadIfAbsent( + qx, + purl, + '2026-06-01', + '2026-07-01', + 100, + true, + ) + + expect(changed).toEqual( + expect.arrayContaining(['downloads_last_30d.start_date', 'downloads_last_30d.count']), + ) + expect(changed).toContain('packages.downloads_last_30d') + + const row = await qx.selectOne(`SELECT count FROM downloads_last_30d WHERE purl = $(purl)`, { + purl, + }) + expect(Number(row.count)).toBe(100) + const pkg = await qx.selectOne(`SELECT downloads_last_30d FROM packages WHERE purl = $(purl)`, { + purl, + }) + expect(Number(pkg.downloads_last_30d)).toBe(100) + }) + + it('does not overwrite an existing window — later call with a different count is a no-op', async () => { + const purl = `pkg:composer/${FIXTURE_TAG}/second-call-loses` + await makePackage(purl) + + const first = await insertLast30dDownloadIfAbsent( + qx, + purl, + '2026-06-01', + '2026-07-01', + 100, + true, + ) + expect(first.length).toBeGreaterThan(0) + + const second = await insertLast30dDownloadIfAbsent( + qx, + purl, + '2026-06-01', + '2026-07-01', + 999, + true, + ) + expect(second).toEqual([]) + + const row = await qx.selectOne(`SELECT count FROM downloads_last_30d WHERE purl = $(purl)`, { + purl, + }) + expect(Number(row.count)).toBe(100) + const pkg = await qx.selectOne(`SELECT downloads_last_30d FROM packages WHERE purl = $(purl)`, { + purl, + }) + expect(Number(pkg.downloads_last_30d)).toBe(100) + }) + + it('resolves a genuine race between concurrent callers to exactly one consistent winner', async () => { + const purl = `pkg:composer/${FIXTURE_TAG}/concurrent-race` + await makePackage(purl) + + const [a, b] = await Promise.all([ + insertLast30dDownloadIfAbsent(qx, purl, '2026-06-01', '2026-07-01', 111, true), + insertLast30dDownloadIfAbsent(qx, purl, '2026-06-01', '2026-07-01', 222, true), + ]) + + // exactly one of the two concurrent callers observes itself as the writer + const winners = [a, b].filter((c) => c.length > 0) + expect(winners).toHaveLength(1) + + const row = await qx.selectOne(`SELECT count FROM downloads_last_30d WHERE purl = $(purl)`, { + purl, + }) + const pkg = await qx.selectOne(`SELECT downloads_last_30d FROM packages WHERE purl = $(purl)`, { + purl, + }) + // whichever value won, the window row and its packages mirror must agree + expect(Number(pkg.downloads_last_30d)).toBe(Number(row.count)) + expect([111, 222]).toContain(Number(row.count)) + }) +}) diff --git a/services/libs/data-access-layer/src/packages/downloadsLast30d.ts b/services/libs/data-access-layer/src/packages/downloadsLast30d.ts index 8ac1e3e718..6189d4bf2b 100644 --- a/services/libs/data-access-layer/src/packages/downloadsLast30d.ts +++ b/services/libs/data-access-layer/src/packages/downloadsLast30d.ts @@ -133,6 +133,48 @@ export async function markLast30dHistoryBackfilled( ) } +// Atomic "first observation wins" insert: the window row and its packages mirror are +// written by one statement via ON CONFLICT (purl, end_date) DO NOTHING, so concurrent +// callers for the same purl+month (overlapping Temporal retries, or two lanes racing) +// can never both pass a check-then-insert gap and have the later one clobber the +// earlier one's count — only the writer that actually inserts the row also mirrors. +export async function insertLast30dDownloadIfAbsent( + qx: QueryExecutor, + purl: string, + startDate: string, + endDate: string, + count: number, + mirrorToPackages: boolean, +): Promise { + const row: { inserted: boolean; mirrored: boolean } = await qx.selectOne( + `WITH ins AS ( + INSERT INTO downloads_last_30d (purl, start_date, end_date, count, created_at, updated_at) + VALUES ($(purl), $(startDate)::date, $(endDate)::date, $(count), NOW(), NOW()) + ON CONFLICT (purl, end_date) DO NOTHING + RETURNING purl, count + ), + pkg AS ( + UPDATE packages p + SET downloads_last_30d = ins.count, + last_synced_at = NOW() + FROM ins + WHERE p.purl = ins.purl + AND $(mirrorToPackages) + AND p.downloads_last_30d IS DISTINCT FROM ins.count + RETURNING p.id + ) + SELECT + EXISTS (SELECT 1 FROM ins) AS inserted, + EXISTS (SELECT 1 FROM pkg) AS mirrored`, + { purl, startDate, endDate, count, mirrorToPackages }, + ) + + if (!row.inserted) return [] + const changed = ['downloads_last_30d.start_date', 'downloads_last_30d.count'] + if (row.mirrored) changed.push('packages.downloads_last_30d') + return changed +} + export async function upsertLast30dDownload( qx: QueryExecutor, purl: string, @@ -163,9 +205,12 @@ export async function upsertLast30dDownload( ) const changed = row.changed_fields if (mirrorToPackages) { + // last_synced_at is the Tinybird ENGINE_VER for the packages datasource — it must + // move whenever a real column changes, or Tinybird can keep serving a stale row. const rowCount = await qx.result( `UPDATE packages - SET downloads_last_30d = $(count) + SET downloads_last_30d = $(count), + last_synced_at = NOW() WHERE purl = $(purl) AND downloads_last_30d IS DISTINCT FROM $(count)`, { count, purl }, ) diff --git a/services/libs/data-access-layer/src/packages/index.ts b/services/libs/data-access-layer/src/packages/index.ts index 13631858c0..00fca97d7b 100644 --- a/services/libs/data-access-layer/src/packages/index.ts +++ b/services/libs/data-access-layer/src/packages/index.ts @@ -1,10 +1,12 @@ export * from './auditFieldChanges' +export * from './dependencies' export * from './downloadsDaily' export * from './downloadsLast30d' export * from './fundingLinks' export * from './maintainers' export * from './npmPackageState' export * from './npmWorkerState' +export * from './packagistPackageState' export * from './pypiPackageState' export * from './packages' export * from './repos' diff --git a/services/libs/data-access-layer/src/packages/packages.ts b/services/libs/data-access-layer/src/packages/packages.ts index 3a18d0cd82..ea987f9b51 100644 --- a/services/libs/data-access-layer/src/packages/packages.ts +++ b/services/libs/data-access-layer/src/packages/packages.ts @@ -226,6 +226,176 @@ export async function getTrackedNpmPackages( })) } +export interface PackagistSeedRow { + purl: string + vendor: string + name: string +} + +export async function insertPackagistPackages( + qx: QueryExecutor, + rows: PackagistSeedRow[], +): Promise { + if (rows.length === 0) return 0 + const purls = rows.map((r) => r.purl) + const vendors = rows.map((r) => r.vendor) + const names = rows.map((r) => r.name) + const result = await qx.result( + `INSERT INTO packages (purl, ecosystem, namespace, name, registry_url, status, ingestion_source, last_synced_at, created_at) + SELECT r.purl, 'packagist', r.vendor, r.name, + 'https://packagist.org/packages/' || r.vendor || '/' || r.name, + 'active', 'packagist-registry', NOW(), NOW() + FROM unnest($(purls)::text[], $(vendors)::text[], $(names)::text[]) + AS r(purl, vendor, name) + ON CONFLICT (purl) DO NOTHING`, + { purls, vendors, names }, + ) + return result +} + +export interface PackagistStatsUpdateInput { + purl: string + description: string | null + declaredRepositoryUrl: string | null + repositoryUrl: string | null + status: string + totalDownloads: number | null + dependentCount: number | null +} + +// downloads_last_30d is NOT written here — it belongs exclusively to the dedicated +// downloads-30d lane's boundary-anchored, insert-if-absent snapshot. The metadata +// lane's live rolling `downloads.monthly` value is a different, unanchored reading; +// writing it here would silently overwrite that snapshot every weekly refresh. +export async function updatePackagistPackageStats( + qx: QueryExecutor, + input: PackagistStatsUpdateInput, +): Promise<{ id: string; isCritical: boolean; changedFields: string[] } | null> { + const row: { id: string; is_critical: boolean; changed_fields: string[] } | undefined = + await qx.selectOneOrNone( + `WITH old AS ( + SELECT description, declared_repository_url, repository_url, status, + total_downloads, dependent_count, ingestion_source + FROM packages WHERE purl = $(purl) AND ecosystem = 'packagist' + ), + ins AS ( + UPDATE packages SET + description = $(description), + declared_repository_url = $(declaredRepositoryUrl), + repository_url = $(repositoryUrl), + status = $(status), + total_downloads = COALESCE($(totalDownloads), total_downloads), + dependent_count = COALESCE($(dependentCount), dependent_count), + last_synced_at = NOW() + WHERE purl = $(purl) AND ecosystem = 'packagist' + RETURNING id, is_critical, description, declared_repository_url, repository_url, status, + total_downloads, dependent_count, ingestion_source + ) + SELECT ins.id::text AS id, ins.is_critical, + array_remove(ARRAY[ + CASE WHEN o.description IS DISTINCT FROM ins.description THEN 'packages.description' END, + CASE WHEN o.declared_repository_url IS DISTINCT FROM ins.declared_repository_url THEN 'packages.declared_repository_url' END, + CASE WHEN o.repository_url IS DISTINCT FROM ins.repository_url THEN 'packages.repository_url' END, + CASE WHEN o.status IS DISTINCT FROM ins.status THEN 'packages.status' END, + CASE WHEN o.total_downloads IS DISTINCT FROM ins.total_downloads THEN 'packages.total_downloads' END, + CASE WHEN o.dependent_count IS DISTINCT FROM ins.dependent_count THEN 'packages.dependent_count' END, + CASE WHEN o.ingestion_source IS DISTINCT FROM ins.ingestion_source THEN 'packages.ingestion_source' END + ], NULL) AS changed_fields + FROM ins LEFT JOIN old o ON true`, + input, + ) + + if (!row) return null + return { id: row.id, isCritical: row.is_critical, changedFields: row.changed_fields } +} + +export interface PackagistVersionAggregates { + versionsCount: number + latestVersion: string | null + firstReleaseAt: string | null + latestReleaseAt: string | null + licenses: string[] | null + homepage: string | null +} + +export async function updatePackagistVersionAggregates( + qx: QueryExecutor, + purl: string, + agg: PackagistVersionAggregates, +): Promise<{ id: string; changedFields: string[] } | null> { + const row: { id: string; changed_fields: string[] } | undefined = await qx.selectOneOrNone( + `WITH old AS ( + SELECT versions_count, latest_version, first_release_at, latest_release_at, licenses, homepage, ingestion_source + FROM packages WHERE purl = $(purl) AND ecosystem = 'packagist' + ), + ins AS ( + UPDATE packages SET + versions_count = CASE WHEN $(versionsCount) = 0 THEN packages.versions_count ELSE $(versionsCount) END, + -- keep-on-zero like versions_count: a dev-only manifest must not null the + -- known latest while the count is preserved + latest_version = CASE WHEN $(versionsCount) = 0 THEN packages.latest_version ELSE $(latestVersion) END, + first_release_at = COALESCE($(firstReleaseAt), first_release_at), + latest_release_at = COALESCE($(latestReleaseAt), latest_release_at), + licenses = COALESCE($(licenses), licenses), + homepage = COALESCE($(homepage), homepage), + last_synced_at = NOW() + WHERE purl = $(purl) AND ecosystem = 'packagist' + RETURNING id, versions_count, latest_version, first_release_at, latest_release_at, licenses, homepage, ingestion_source + ) + SELECT ins.id::text AS id, + array_remove(ARRAY[ + CASE WHEN o.versions_count IS DISTINCT FROM ins.versions_count THEN 'packages.versions_count' END, + CASE WHEN o.latest_version IS DISTINCT FROM ins.latest_version THEN 'packages.latest_version' END, + CASE WHEN o.first_release_at IS DISTINCT FROM ins.first_release_at THEN 'packages.first_release_at' END, + CASE WHEN o.latest_release_at IS DISTINCT FROM ins.latest_release_at THEN 'packages.latest_release_at' END, + CASE WHEN o.licenses IS DISTINCT FROM ins.licenses THEN 'packages.licenses' END, + CASE WHEN o.homepage IS DISTINCT FROM ins.homepage THEN 'packages.homepage' END, + CASE WHEN o.ingestion_source IS DISTINCT FROM ins.ingestion_source THEN 'packages.ingestion_source' END + ], NULL) AS changed_fields + FROM ins LEFT JOIN old o ON true`, + { purl, ...agg }, + ) + + if (!row) return null + return { id: row.id, changedFields: row.changed_fields } +} + +export async function getPackagistPackageIdsByNames( + qx: QueryExecutor, + names: string[], +): Promise> { + // Composer names are case-insensitive but seed rows are stored lowercased — + // lowercase the lookup so a manifest declaring 'Monolog/Monolog' still resolves. + const lowerByOriginal = new Map(names.map((name) => [name, name.toLowerCase()])) + const purls = [...lowerByOriginal.values()].map((name) => `pkg:composer/${name}`) + + const rows: Array<{ purl: string; id: string }> = await qx.select( + `SELECT purl, id::text AS id FROM packages + WHERE ecosystem = 'packagist' AND purl = ANY($(purls)::text[])`, + { purls }, + ) + + const idByLower = new Map() + for (const row of rows) { + idByLower.set(row.purl.slice('pkg:composer/'.length), row.id) + } + + // Map back keyed by the ORIGINAL (caller-supplied) names + const map = new Map() + for (const [original, lower] of lowerByOriginal) { + const id = idByLower.get(lower) + if (id) map.set(original, id) + } + return map +} + +export async function getCriticalPackagistPackageCount(qx: QueryExecutor): Promise { + const row: { count: string } = await qx.selectOne( + `SELECT COUNT(*)::text AS count FROM packages WHERE ecosystem = 'packagist' AND is_critical = TRUE`, + ) + return Number(row.count) +} + // How many critical PyPI packages exist — a cheap guard so the daily downloads workflow can // skip its BigQuery scan entirely when there are none to ingest (the merge scopes to is_critical). export async function getCriticalPypiPackageCount(qx: QueryExecutor): Promise { diff --git a/services/libs/data-access-layer/src/packages/packagistPackageState.ts b/services/libs/data-access-layer/src/packages/packagistPackageState.ts new file mode 100644 index 0000000000..8d734a2474 --- /dev/null +++ b/services/libs/data-access-layer/src/packages/packagistPackageState.ts @@ -0,0 +1,186 @@ +import { QueryExecutor } from '../queryExecutor' + +// Structured outcome of a packagist ingest run, stored as JSONB in +// packagist_package_state.{metadata,downloads_30d,daily_downloads}_run_result. +export interface PackagistRunResult { + status: 'success' | 'error' + attempts: number + httpStatus?: number + errorKind?: string + message?: string +} + +export interface PackagistMetadataCandidate { + purl: string + // Last-Modified from the previous p2 fetch, replayed as If-Modified-Since. + metadataLastModified: string | null +} + +export interface PackagistDailyCandidate { + purl: string + packageId: string +} + +export interface MarkMetadataScannedOptions { + metadataLastModified?: string | null + // p2-only failures (phase 1 already succeeded) should NOT push the refresh watermark + // forward — versions/dependencies never actually refreshed, so the package must stay + // (or become) due again on the next run instead of sitting out the full refresh window. + bumpLastRunAt?: boolean + // Give-up (error) writes only: this activity's own scheduledTimestampMs (stable across + // Temporal retries of the same batch). Guards against a spurious re-processing of an + // item that already succeeded earlier in this same batch's retry sequence overwriting + // that success with an error — only blocks the write if the stored success happened at + // or after this activity was first scheduled, so a genuinely new failure on a later, + // unrelated run still always records normally. + notBefore?: string | null +} + +export async function markPackagistMetadataScanned( + qx: QueryExecutor, + purl: string, + result: PackagistRunResult, + options: MarkMetadataScannedOptions = {}, +): Promise { + const { metadataLastModified = null, bumpLastRunAt = true, notBefore = null } = options + await qx.result( + `INSERT INTO packagist_package_state (purl, metadata_run_result, metadata_last_run_at, metadata_last_modified) + VALUES ($(purl), $(result)::jsonb, CASE WHEN $(bumpLastRunAt) THEN NOW() ELSE NULL END, $(metadataLastModified)) + ON CONFLICT (purl) DO UPDATE SET + metadata_run_result = EXCLUDED.metadata_run_result, + metadata_last_run_at = CASE WHEN $(bumpLastRunAt) THEN EXCLUDED.metadata_last_run_at + ELSE packagist_package_state.metadata_last_run_at END, + metadata_last_modified = COALESCE(EXCLUDED.metadata_last_modified, packagist_package_state.metadata_last_modified) + WHERE $(notBefore)::timestamptz IS NULL + OR NOT ( + packagist_package_state.metadata_run_result->>'status' = 'success' + AND packagist_package_state.metadata_last_run_at >= $(notBefore)::timestamptz + )`, + { + purl, + result: JSON.stringify(result), + metadataLastModified, + bumpLastRunAt, + notBefore, + }, + ) +} + +export async function getPackagistMetadataDuePurls( + qx: QueryExecutor, + cutoff: string, + afterPurl: string, + batchSize: number, + onlyCritical: boolean, +): Promise { + const rows: Array<{ purl: string; metadata_last_modified: string | null }> = await qx.select( + `SELECT p.purl, s.metadata_last_modified + FROM packages p + LEFT JOIN packagist_package_state s ON s.purl = p.purl + WHERE p.ecosystem = 'packagist' + AND (NOT $(onlyCritical) OR p.is_critical = TRUE) + AND p.purl > $(afterPurl) + AND ( + s.metadata_last_run_at IS NULL + OR s.metadata_last_run_at < $(cutoff)::timestamptz + ) + ORDER BY p.purl + LIMIT $(batchSize)`, + { cutoff, afterPurl, batchSize, onlyCritical }, + ) + return rows.map((r) => ({ + purl: r.purl, + metadataLastModified: r.metadata_last_modified, + })) +} + +export async function getPackagist30dDuePurls( + qx: QueryExecutor, + cutoff: string, + afterPurl: string, + batchSize: number, +): Promise { + const rows: Array<{ purl: string }> = await qx.select( + `SELECT p.purl + FROM packages p + LEFT JOIN packagist_package_state s ON s.purl = p.purl + WHERE p.ecosystem = 'packagist' + AND p.purl > $(afterPurl) + AND ( + s.downloads_30d_last_run_at IS NULL + OR s.downloads_30d_last_run_at < $(cutoff)::timestamptz + ) + ORDER BY p.purl + LIMIT $(batchSize)`, + { cutoff, afterPurl, batchSize }, + ) + return rows.map((r) => r.purl) +} + +export async function markPackagist30dProcessed( + qx: QueryExecutor, + purl: string, + result: PackagistRunResult, + // Give-up (error) writes only — see markPackagistMetadataScanned's notBefore for why. + notBefore: string | null = null, +): Promise { + await qx.result( + `INSERT INTO packagist_package_state (purl, downloads_30d_run_result, downloads_30d_last_run_at) + VALUES ($(purl), $(result)::jsonb, NOW()) + ON CONFLICT (purl) DO UPDATE SET + downloads_30d_run_result = EXCLUDED.downloads_30d_run_result, + downloads_30d_last_run_at = EXCLUDED.downloads_30d_last_run_at + WHERE $(notBefore)::timestamptz IS NULL + OR NOT ( + packagist_package_state.downloads_30d_run_result->>'status' = 'success' + AND packagist_package_state.downloads_30d_last_run_at >= $(notBefore)::timestamptz + )`, + { purl, result: JSON.stringify(result), notBefore }, + ) +} + +export async function getPackagistDailyDownloadsDue( + qx: QueryExecutor, + cutoff: string, + afterPurl: string, + batchSize: number, +): Promise { + const rows: Array<{ purl: string; package_id: string }> = await qx.select( + `SELECT p.purl, p.id::text AS package_id + FROM packages p + LEFT JOIN packagist_package_state s ON s.purl = p.purl + WHERE p.ecosystem = 'packagist' + AND p.is_critical = TRUE + AND p.purl > $(afterPurl) + AND ( + s.daily_downloads_last_run_at IS NULL + OR s.daily_downloads_last_run_at < $(cutoff)::timestamptz + ) + ORDER BY p.purl + LIMIT $(batchSize)`, + { cutoff, afterPurl, batchSize }, + ) + return rows.map((r) => ({ purl: r.purl, packageId: r.package_id })) +} + +export async function markPackagistDailyProcessed( + qx: QueryExecutor, + purl: string, + result: PackagistRunResult, + // Give-up (error) writes only — see markPackagistMetadataScanned's notBefore for why. + notBefore: string | null = null, +): Promise { + await qx.result( + `INSERT INTO packagist_package_state (purl, daily_downloads_run_result, daily_downloads_last_run_at) + VALUES ($(purl), $(result)::jsonb, NOW()) + ON CONFLICT (purl) DO UPDATE SET + daily_downloads_run_result = EXCLUDED.daily_downloads_run_result, + daily_downloads_last_run_at = EXCLUDED.daily_downloads_last_run_at + WHERE $(notBefore)::timestamptz IS NULL + OR NOT ( + packagist_package_state.daily_downloads_run_result->>'status' = 'success' + AND packagist_package_state.daily_downloads_last_run_at >= $(notBefore)::timestamptz + )`, + { purl, result: JSON.stringify(result), notBefore }, + ) +} diff --git a/services/libs/data-access-layer/src/packages/repos.ts b/services/libs/data-access-layer/src/packages/repos.ts index 469ce883a8..92d14a55b8 100644 --- a/services/libs/data-access-layer/src/packages/repos.ts +++ b/services/libs/data-access-layer/src/packages/repos.ts @@ -32,6 +32,28 @@ export async function getOrCreateRepoByUrl( return { id: row.id, changedFields: [] } } +// Removes a package's previous 'declared' repo link(s) when its manifest no longer +// resolves to a trusted repo (the field was removed, or no longer canonicalizes), or +// now resolves to a different one (pass exceptRepoId to keep only that fresh link — +// package_repos' unique key is (package_id, repo_id), not (package_id, source), so a +// plain upsert of the new link never removes a stale one pointing elsewhere). Only +// touches 'declared' — other sources (deps_dev, heuristic, manual) are owned by +// different pipelines and left alone. +export async function removeDeclaredPackageRepo( + qx: QueryExecutor, + packageId: string, + exceptRepoId?: string, +): Promise { + const rowCount = await qx.result( + `DELETE FROM package_repos + WHERE package_id = $(packageId)::bigint + AND source = 'declared' + AND ($(exceptRepoId)::bigint IS NULL OR repo_id <> $(exceptRepoId)::bigint)`, + { packageId, exceptRepoId: exceptRepoId ?? null }, + ) + return rowCount > 0 ? ['package_repos.repo_id'] : [] +} + export async function upsertPackageRepo( qx: QueryExecutor, packageId: string, @@ -65,3 +87,42 @@ export async function upsertPackageRepo( ) return row.changed_fields } + +// Same shape as upsertPackageRepo, but never lets this write downgrade an existing +// link: `source` is left untouched on conflict (a manual/deps_dev link keeps its +// provenance instead of being reassigned to this caller's source) and `confidence` +// only ever moves up via GREATEST. Matches the established pattern in +// upsertMavenPackageRepo (osspckgs/repos.ts) and cargo/enrich.ts's inline equivalent — +// added here as a separate function rather than changing upsertPackageRepo itself, +// which pypi/npm/go/nuget/rubygems also call and rely on staying an unconditional set. +export async function upsertPackageRepoPreserveProvenance( + qx: QueryExecutor, + packageId: string, + repoId: string, + source: string, + confidence: number, +): Promise { + const row: { changed_fields: string[] } = await qx.selectOne( + `WITH old AS ( + SELECT source, confidence FROM package_repos + WHERE package_id = $(packageId)::bigint AND repo_id = $(repoId)::bigint + ), + ins AS ( + INSERT INTO package_repos (package_id, repo_id, source, confidence, created_at) + VALUES ($(packageId)::bigint, $(repoId)::bigint, $(source), $(confidence), NOW()) + ON CONFLICT (package_id, repo_id) DO UPDATE SET + confidence = GREATEST(EXCLUDED.confidence, package_repos.confidence), + verified_at = NOW() + RETURNING source, confidence + ) + SELECT array_remove(ARRAY[ + CASE WHEN o.source IS NULL THEN 'package_repos.repo_id' END, + CASE WHEN o.source IS NULL THEN 'package_repos.source' END, + CASE WHEN o.source IS NULL + OR o.confidence IS DISTINCT FROM ins.confidence THEN 'package_repos.confidence' END + ], NULL) AS changed_fields + FROM ins LEFT JOIN old o ON true`, + { packageId, repoId, source, confidence }, + ) + return row.changed_fields +} diff --git a/services/libs/data-access-layer/src/packages/versions.ts b/services/libs/data-access-layer/src/packages/versions.ts index f3fff47bc8..f734709503 100644 --- a/services/libs/data-access-layer/src/packages/versions.ts +++ b/services/libs/data-access-layer/src/packages/versions.ts @@ -69,6 +69,112 @@ export async function upsertNpmVersions( return row.changed_fields } +export interface PackagistVersionInput { + number: string + publishedAt: string | null + isLatest: boolean + isPrerelease: boolean + // Composer allows dual/multi-licensed releases (e.g. ["MIT", "Apache-2.0"]) — the + // full array is preserved, unlike the single-SPDX-string npm/pypi license inputs. + licenses: string[] | null +} + +export async function upsertPackagistVersions( + qx: QueryExecutor, + packageId: string, + versions: PackagistVersionInput[], + latestNumber: string | null, +): Promise<{ changedFields: string[]; versionIds: Array<{ number: string; id: string }> }> { + if (versions.length === 0) return { changedFields: [], versionIds: [] } + + const row: { + changed_fields: string[] + version_ids: Array<{ number: string; id: string }> + } = await qx.selectOne( + `WITH old AS ( + SELECT number, published_at, is_latest, is_prerelease, licenses + FROM versions + WHERE package_id = $(packageId)::bigint AND number = ANY($(numbers)::text[]) + ), + ins AS ( + INSERT INTO versions ( + package_id, ecosystem, namespace, name, number, + published_at, is_latest, is_prerelease, licenses, last_synced_at, + created_at + ) + SELECT $(packageId)::bigint, 'packagist', p.namespace, p.name, v.num, + v.pub::timestamptz, v.latest, v.pre, + -- licenses travels as one JSON-encoded array per row (unnest can't carry a + -- ragged array-of-arrays alongside scalar columns), decoded back here. + CASE WHEN v.lic IS NULL THEN NULL::text[] + ELSE (SELECT array_agg(elem) FROM jsonb_array_elements_text(v.lic::jsonb) AS elem) + END, + NOW(), NOW() + FROM unnest( + $(numbers)::text[], + $(publishedAts)::text[], + $(isLatests)::bool[], + $(isPrereleases)::bool[], + $(licenses)::text[] + ) AS v(num, pub, latest, pre, lic) + CROSS JOIN (SELECT namespace, name FROM packages WHERE id = $(packageId)::bigint) p + ON CONFLICT (package_id, number) DO UPDATE SET + published_at = COALESCE(EXCLUDED.published_at, versions.published_at), + is_latest = EXCLUDED.is_latest, + is_prerelease = EXCLUDED.is_prerelease, + licenses = EXCLUDED.licenses, + last_synced_at = EXCLUDED.last_synced_at + RETURNING number, published_at, is_latest, is_prerelease, licenses, id::text AS id + ) + SELECT array_remove(ARRAY[ + CASE WHEN bool_or(o.number IS NULL) THEN 'versions.number' END, + CASE WHEN bool_or(o.number IS NULL OR o.published_at IS DISTINCT FROM ins.published_at) THEN 'versions.published_at' END, + CASE WHEN bool_or(o.is_latest IS DISTINCT FROM ins.is_latest) THEN 'versions.is_latest' END, + CASE WHEN bool_or(o.is_prerelease IS DISTINCT FROM ins.is_prerelease) THEN 'versions.is_prerelease' END, + CASE WHEN bool_or(o.licenses IS DISTINCT FROM ins.licenses) THEN 'versions.licenses' END + ], NULL) AS changed_fields, + json_agg(json_build_object('number', ins.number, 'id', ins.id))::jsonb AS version_ids + FROM ins LEFT JOIN old o ON o.number = ins.number`, + { + packageId, + numbers: versions.map((v) => v.number), + publishedAts: versions.map((v) => v.publishedAt), + isLatests: versions.map((v) => v.isLatest), + isPrereleases: versions.map((v) => v.isPrerelease), + licenses: versions.map((v) => + v.licenses && v.licenses.length > 0 ? JSON.stringify(v.licenses) : null, + ), + }, + ) + + // Parse the version_ids from JSONB + const versionIds = (row.version_ids as Array<{ number: string; id: string }>) || [] + + // Clear a stale is_latest on every OTHER version of this package. Anchored on the declared + // latest (latestNumber) — NOT on what's in this batch — so a previously-latest version whose + // records were updated differently can't keep is_latest = true alongside the new latest. When + // no latest is known, leave flags untouched rather than wipe all. last_synced_at must move too + // — it's the Tinybird ENGINE_VER, and this row's is_latest is a real change. + const changedFields = [...row.changed_fields] + if (latestNumber != null) { + const cleared = await qx.result( + `UPDATE versions SET is_latest = false, last_synced_at = NOW() + WHERE package_id = $(packageId)::bigint + AND is_latest = true + AND number <> $(latestNumber)`, + { packageId, latestNumber }, + ) + // The cleanup only ever flips rows OUTSIDE this batch (batch rows already got their + // is_latest set by the upsert above, so `is_latest = true` skips them) — the CTE diff + // can't see these changes, so surface them for the audit log here. + if (cleared > 0 && !changedFields.includes('versions.is_latest')) { + changedFields.push('versions.is_latest') + } + } + + return { changedFields, versionIds } +} + export interface PypiVersionInput { number: string publishedAt: string | null