Skip to content

Commit e0c45ea

Browse files
zhangkaihua88zhangkaihua88
committed
feat: add private version registration worker
1 parent bb45355 commit e0c45ea

15 files changed

Lines changed: 2027 additions & 1 deletion

‎.github/workflows/telemetry-staging-smoke.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
name: Telemetry staging smoke test
2+
3+
on:
4+
push:
5+
paths:
6+
- 'telemetry-worker/**'
7+
- '.github/workflows/telemetry-staging-smoke.yml'
8+
workflow_dispatch:
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
unit-and-staging:
15+
runs-on: ubuntu-latest
16+
defaults:
17+
run:
18+
working-directory: telemetry-worker
19+
20+
steps:
21+
- name: Checkout code
22+
uses: actions/checkout@v4
23+
24+
- name: Set up Node.js
25+
uses: actions/setup-node@v4
26+
with:
27+
node-version: '24'
28+
29+
- name: Run unit tests
30+
run: node --test
31+
32+
- name: Verify deployed staging Worker
33+
env:
34+
BASE_URL: https://webdatascope-telemetry-staging.zkhweb.workers.dev
35+
run: |
36+
set -euo pipefail
37+
38+
unauthorized=$(curl --retry 3 --retry-all-errors --connect-timeout 10 --max-time 30 \
39+
-sS -o /dev/null -w '%{http_code}' "$BASE_URL/admin")
40+
test "$unauthorized" = "401"
41+
42+
payload='{"schemaVersion":1,"installationId":"323e4567-e89b-42d3-a456-426614174002","wqId":"SYNTHETIC-GITHUB-STAGING-ONLY","country":"GB","version":"1.3.0","previousVersion":"1.2.2","reason":"update"}'
43+
first_file=$(mktemp)
44+
first_status=$(curl --retry 3 --retry-all-errors --connect-timeout 10 --max-time 30 \
45+
-sS -o "$first_file" -w '%{http_code}' -X POST "$BASE_URL/v1/registrations" \
46+
-H 'Content-Type: application/json' --data-binary "$payload")
47+
test "$first_status" = "201" -o "$first_status" = "200"
48+
jq -e '.ok == true and .schemaVersion == 1' "$first_file" >/dev/null
49+
50+
duplicate_file=$(mktemp)
51+
duplicate_status=$(curl --retry 3 --retry-all-errors --connect-timeout 10 --max-time 30 \
52+
-sS -o "$duplicate_file" -w '%{http_code}' -X POST "$BASE_URL/v1/registrations" \
53+
-H 'Content-Type: application/json' --data-binary "$payload")
54+
test "$duplicate_status" = "200"
55+
jq -e '.ok == true and .created == false' "$duplicate_file" >/dev/null
56+
57+
invalid='{"schemaVersion":1,"installationId":"323e4567-e89b-42d3-a456-426614174002","wqId":"X","country":"GB","version":"1.3.0","previousVersion":null,"reason":"install","pageUrl":"https://private.invalid"}'
58+
invalid_status=$(curl --retry 3 --retry-all-errors --connect-timeout 10 --max-time 30 \
59+
-sS -o /dev/null -w '%{http_code}' -X POST "$BASE_URL/v1/registrations" \
60+
-H 'Content-Type: application/json' --data-binary "$invalid")
61+
test "$invalid_status" = "400"
62+
63+
preflight_status=$(curl --retry 3 --retry-all-errors --connect-timeout 10 --max-time 30 \
64+
-sS -o /dev/null -w '%{http_code}' -X OPTIONS "$BASE_URL/v1/registrations")
65+
test "$preflight_status" = "204"

‎.gitignore‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
raw_data/*
22
data_all/*
3-
node_modules/*
3+
node_modules/
4+
.pnpm-store/
45
Reference/*
56
package-lock.json
67
package.json
8+
!telemetry-worker/package.json
9+
!telemetry-worker/pnpm-lock.yaml
710
data/*
811
figure/*
912
img/screenshot.png
@@ -18,3 +21,7 @@ DEVELOPMENT.md
1821
操作说明.md
1922
.analysis_repos/*
2023
.sidebar-content-src/
24+
telemetry-worker/.wrangler/
25+
telemetry-worker/.dev.vars*
26+
telemetry-worker/.generated-secrets.json
27+
telemetry-worker/.admin-credentials.txt

‎telemetry-worker/README.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# WebDataScope telemetry Worker
2+
3+
This subproject implements the opt-out-by-version registration service announced in extension V1.2.2 and enabled in V1.3.0. It records one row per installation, WQ account and extension version. It does not collect feature events or page activity, so it cannot provide DAU/WAU or usage-frequency metrics.
4+
5+
## Data handling
6+
7+
- The registration endpoint accepts only schema version, installation UUID, WQ ID, country, extension version, previous version and reason.
8+
- Request bodies are limited to 2 KiB and unknown fields are rejected.
9+
- The raw WQ ID is HMAC-indexed and AES-256-GCM encrypted before D1 writes. D1 never stores it as plaintext.
10+
- WorldQuant cookies never reach this Worker. Extension requests use `credentials: "omit"`.
11+
- Cloudflare's connecting IP is used only as an ephemeral rate-limit key and is not written to D1.
12+
- Worker code does not log request bodies, WQ IDs or decrypted values.
13+
- `COLLECTION_ENABLED=false` is the emergency stop.
14+
15+
Because the same browser installation may switch WQ accounts, the database idempotency key is `(installation_id, account_hash, version)`. This is the smallest key that both prevents duplicate retries and preserves the required account-switch record.
16+
17+
## Local verification
18+
19+
1. Install dependencies with `pnpm install`.
20+
2. Run `pnpm run secrets:generate`; the ignored files `.generated-secrets.json`, `.dev.vars` and `.admin-credentials.txt` are created without printing secret values.
21+
3. Apply `pnpm run db:migrate:local`.
22+
4. Run `pnpm test` and `pnpm run check`.
23+
24+
## Deployment order
25+
26+
1. Create separate D1 databases named `webdatascope-telemetry-staging` and `webdatascope-telemetry`.
27+
2. Replace the placeholder database IDs in `wrangler.jsonc`.
28+
3. Upload `.generated-secrets.json` with `wrangler secret bulk` to staging and production. Never commit it.
29+
4. Apply remote migrations to staging, deploy staging, and perform the synthetic registration/admin/deletion checks.
30+
5. Apply remote migrations to production, deploy production, repeat the checks, then place the production `/v1/registrations` URL in the extension.
31+
32+
The administrator credentials are stored only in the ignored `.admin-credentials.txt`. Rotate them by generating a new password, updating `ADMIN_AUTH_DIGEST`, and securely retaining the new local credential.

‎telemetry-worker/migrations/0001_initial.sql‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
PRAGMA foreign_keys = ON;
2+
3+
CREATE TABLE IF NOT EXISTS accounts (
4+
account_hash TEXT PRIMARY KEY,
5+
encrypted_wq_id TEXT NOT NULL,
6+
encryption_iv TEXT NOT NULL,
7+
key_version INTEGER NOT NULL,
8+
country TEXT NOT NULL,
9+
latest_version TEXT NOT NULL,
10+
latest_version_rank TEXT NOT NULL,
11+
first_seen_at TEXT NOT NULL,
12+
last_seen_at TEXT NOT NULL
13+
);
14+
15+
CREATE TABLE IF NOT EXISTS installations (
16+
installation_id TEXT NOT NULL,
17+
account_hash TEXT NOT NULL,
18+
first_seen_at TEXT NOT NULL,
19+
last_seen_at TEXT NOT NULL,
20+
PRIMARY KEY (installation_id, account_hash),
21+
FOREIGN KEY (account_hash) REFERENCES accounts(account_hash) ON DELETE CASCADE
22+
);
23+
24+
CREATE TABLE IF NOT EXISTS version_registrations (
25+
installation_id TEXT NOT NULL,
26+
account_hash TEXT NOT NULL,
27+
version TEXT NOT NULL,
28+
previous_version TEXT,
29+
reason TEXT NOT NULL CHECK (reason IN ('install', 'update', 'retry')),
30+
country TEXT NOT NULL,
31+
first_reported_at TEXT NOT NULL,
32+
PRIMARY KEY (installation_id, account_hash, version),
33+
FOREIGN KEY (installation_id, account_hash)
34+
REFERENCES installations(installation_id, account_hash)
35+
ON DELETE CASCADE
36+
);
37+
38+
CREATE INDEX IF NOT EXISTS idx_accounts_country ON accounts(country);
39+
CREATE INDEX IF NOT EXISTS idx_accounts_latest_version ON accounts(latest_version);
40+
CREATE INDEX IF NOT EXISTS idx_accounts_last_seen ON accounts(last_seen_at DESC);
41+
CREATE INDEX IF NOT EXISTS idx_installations_account ON installations(account_hash);
42+
CREATE INDEX IF NOT EXISTS idx_registrations_account ON version_registrations(account_hash);
43+
CREATE INDEX IF NOT EXISTS idx_registrations_reported ON version_registrations(first_reported_at DESC);
44+
CREATE INDEX IF NOT EXISTS idx_registrations_upgrade ON version_registrations(previous_version, version);

‎telemetry-worker/package.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"name": "webdatascope-telemetry-worker",
3+
"version": "1.0.0",
4+
"private": true,
5+
"type": "module",
6+
"scripts": {
7+
"test": "node --test",
8+
"dev": "wrangler dev --local --port 8788",
9+
"check": "wrangler deploy --dry-run --env=\"\"",
10+
"types": "wrangler types",
11+
"db:migrate:local": "wrangler d1 migrations apply DB --local",
12+
"db:migrate:staging": "wrangler d1 migrations apply DB --env staging --remote",
13+
"db:migrate:production": "wrangler d1 migrations apply DB --remote",
14+
"secrets:generate": "node scripts/generate-secrets.mjs"
15+
},
16+
"devDependencies": {
17+
"wrangler": "^4.36.0"
18+
}
19+
}

0 commit comments

Comments
 (0)