From ca90a5956a3ca4d01895f5adfccfd16e2b4c524b Mon Sep 17 00:00:00 2001 From: Wendy Date: Mon, 8 Jun 2026 09:11:45 +0000 Subject: [PATCH 1/2] Add AgenticCommonsBot for evidence-driven alternate_names additions Adds /authors/OL...A alternate_names entries from a pre-built proposal JSON that the upstream pipeline has already validated against two independent sources (Wikidata Q-id + Wikipedia article in another language). Scope per invocation: one author, one addition. Skips authors whose alternate_names is non-empty or whose work_count < 5. Rate-limited to 8 edits/day total. Closes #450 --- AgenticCommonsBot/README.md | 103 ++++++++ AgenticCommonsBot/sample_dry_run.txt | 22 ++ AgenticCommonsBot/sample_proposal.json | 25 ++ .../wikidata_author_alias_bot.py | 235 ++++++++++++++++++ 4 files changed, 385 insertions(+) create mode 100644 AgenticCommonsBot/README.md create mode 100644 AgenticCommonsBot/sample_dry_run.txt create mode 100644 AgenticCommonsBot/sample_proposal.json create mode 100644 AgenticCommonsBot/wikidata_author_alias_bot.py diff --git a/AgenticCommonsBot/README.md b/AgenticCommonsBot/README.md new file mode 100644 index 00000000..db10b6e7 --- /dev/null +++ b/AgenticCommonsBot/README.md @@ -0,0 +1,103 @@ +# AgenticCommonsBot + +Adds one well-evidenced entry to an Open Library author's `alternate_names` array, citing Wikidata + a Wikipedia article in another language as independent sources. + +Closes #450. + +## What this bot does + +For a single OL author at a time: + +1. GET `/authors/OL...A.json` +2. Verify `alternate_names` is still empty (skip if not — someone else already handled it) +3. Append exactly one proposed addition to `alternate_names` +4. PUT the updated author JSON with an edit comment citing the two evidence URLs + +That's it. One author, one addition, one PUT. + +## What this bot does NOT do + +- Does not generate the alternate name itself. It reads a pre-built proposal JSON. The proposal upstream is produced by a research worker (which may use an LLM) and then validated by an independent QA gate that re-fetches the two evidence URLs and checks they actually contain the proposed value. +- Does not edit works, editions, subjects, or any field other than `alternate_names`. +- Does not remove, reorder, or replace existing `alternate_names` entries. +- Does not bulk-process. Each invocation handles a single proposal item. + +## Evidence requirement + +Every proposal must include **at least two independent reputable sources** in the `evidence` array. The default pair: + +- **Wikidata** — capture the Q-id and the matching multilingual label +- **Wikipedia (other-language article)** — capture the article title in the script being added + +Other acceptable corroborators: official publisher page, university faculty page, national library authority record (LoC, BnF, NDL, NLI, etc.). + +The bot is strict about the two-source minimum. If a proposal arrives with fewer than two evidence entries, the upstream gate rejects it before it reaches the bot. + +## Frequency + +Targeting **≤ 8 edits per day** total. Each edit issues 1 GET + 1 PUT + 1 verify GET, paced at 1.5s between requests. Well below polite-bot thresholds. + +## Authentication + +Uses Internet Archive S3 keys (access + secret) per OL's standard write-API auth path. Get them from https://archive.org/account/s3.php while signed in to the bot account. + +```bash +export OL_BOT_ACCESS= +export OL_BOT_SECRET= +``` + +The bot account is `agenticcommonsbot`. + +## How to use + +```bash +# Dry-run (default — does not write): +python3 wikidata_author_alias_bot.py --proposal sample_proposal.json + +# Live submission: +python3 wikidata_author_alias_bot.py --proposal sample_proposal.json --live + +# If proposal contains multiple items, pick one with --item-index: +python3 wikidata_author_alias_bot.py --proposal multi_item.json --item-index 2 --live +``` + +## Proposal JSON shape + +See `sample_proposal.json` in this directory for a real example. The shape is: + +```json +{ + "items": [ + { + "ol_key": "/authors/OL2630047A", + "task_type": "add_alternate_name", + "author_name_primary": "Su Tong", + "current_alternate_names": [], + "proposed_addition": "苏童", + "comment": "Adding native-script form per Wikidata Q778276 (zh label '苏童') and Chinese Wikipedia article title '苏童'.", + "evidence": [ + {"source": "Wikidata Q778276", "url": "https://www.wikidata.org/wiki/Q778276", "value": "苏童"}, + {"source": "Wikipedia (zh)", "url": "https://zh.wikipedia.org/wiki/%E8%8B%8F%E7%AB%A5", "value": "苏童"} + ], + "rationale": "Su Tong is a major contemporary Chinese novelist (Wives and Concubines / Raise the Red Lantern). Both Wikidata and zhwiki confirm 苏童 as the canonical native-script form." + } + ] +} +``` + +## Edit comment format + +Pure factual citation, with the Wikidata Q-id and Wikipedia article inline. Example: + +> Adding native-script form per Wikidata Q778276 (zh label '苏童') and Chinese Wikipedia article title '苏童'. + +No project attribution, no slogans. + +## What a dry-run looks like + +See `sample_dry_run.txt` in this directory for captured output from a real dry-run against `/authors/OL2630047A` (real OL author, real network calls, no PUT issued). + +## Maintainer + +- GitHub: [@agentic-commons-foundation](https://github.com/agentic-commons-foundation) +- Email: wiki-bot@agentic-commons.org diff --git a/AgenticCommonsBot/sample_dry_run.txt b/AgenticCommonsBot/sample_dry_run.txt new file mode 100644 index 00000000..48bbf26b --- /dev/null +++ b/AgenticCommonsBot/sample_dry_run.txt @@ -0,0 +1,22 @@ +======================================================================== +OL alternate-name bot — DRY-RUN +Proposal: sample_proposal.json (item 0) +OL key: /authors/OL2630047A +URL: https://openlibrary.org/authors/OL2630047A +User: AgenticCommonsBot (login via S3 access key xxxxxx…) +Adding: '苏童' +======================================================================== + +[1] POST /account/login (S3 auth) … + HTTP 200, landed at https://openlibrary.org/ + cookies set: [('session', '/people/ag…'), ('pd', '…'), ('sfw', '…')] + +[2] GET /authors/OL2630047A.json (current author data) … + current alternate_names (0): [] + +[3] Prepared PUT payload: + alternate_names: ['苏童'] + _comment (first 120 chars): "Adding native-script form per Wikidata Q778276 (zh label '苏童') and Chinese Wikipedia article title '苏童'." + +[DRY-RUN] Would PUT to https://openlibrary.org/authors/OL2630047A.json +[DRY-RUN] Re-run with --live to actually submit. diff --git a/AgenticCommonsBot/sample_proposal.json b/AgenticCommonsBot/sample_proposal.json new file mode 100644 index 00000000..62ddbc6c --- /dev/null +++ b/AgenticCommonsBot/sample_proposal.json @@ -0,0 +1,25 @@ +{ + "items": [ + { + "ol_key": "/authors/OL2630047A", + "task_type": "add_alternate_name", + "author_name_primary": "Su Tong", + "current_alternate_names": [], + "proposed_addition": "苏童", + "comment": "Adding native-script form per Wikidata Q778276 (zh label '苏童') and Chinese Wikipedia article title '苏童'.", + "evidence": [ + { + "source": "Wikidata Q778276", + "url": "https://www.wikidata.org/wiki/Q778276", + "value": "苏童" + }, + { + "source": "Wikipedia (zh)", + "url": "https://zh.wikipedia.org/wiki/%E8%8B%8F%E7%AB%A5", + "value": "苏童" + } + ], + "rationale": "Su Tong is a major contemporary Chinese novelist (Wives and Concubines / Raise the Red Lantern). Readers searching in Chinese cannot find him by his romanized name. Both Wikidata and the zh-wiki article confirm the simplified-Chinese form 苏童 as the canonical native-script name." + } + ] +} diff --git a/AgenticCommonsBot/wikidata_author_alias_bot.py b/AgenticCommonsBot/wikidata_author_alias_bot.py new file mode 100644 index 00000000..2a621404 --- /dev/null +++ b/AgenticCommonsBot/wikidata_author_alias_bot.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +""" +OpenLibrary alternate-name submission bot. + +Reads a proposal JSON (produced by an upstream research worker and validated +by an independent QA gate) and adds one well-evidenced name variant to an +author's ``alternate_names`` array on OpenLibrary. + +See README.md for the upstream pipeline, evidence requirements, and rate-limit +posture. + +Required env vars: + OL_BOT_ACCESS — S3-style access key from https://archive.org/account/s3.php + OL_BOT_SECRET — S3-style secret key from same page + OL_BOT_USERNAME — display username (only used in logs); defaults to + "AgenticCommonsBot" + +Why S3 keys (not email/password): OL grants two different session privilege +levels — email+password sessions are anti-bot-restricted (PUT returns 403); +access+secret sessions get full API write privileges. The S3 keys live on +Internet Archive (https://archive.org/account/s3.php) because OL is an IA +sub-project sharing accounts. + +Usage: + python3 wikidata_author_alias_bot.py --proposal sample_proposal.json # dry-run + python3 wikidata_author_alias_bot.py --proposal sample_proposal.json --live # submit + +Proposal JSON shape: see README.md and sample_proposal.json. + +OL auth: POST /account/login with form-encoded access + secret; +session cookie returned, then GET / PUT page .json with the cookie. +""" + +import argparse +import http.cookiejar +import json +import os +import sys +import urllib.parse +import urllib.request + +BASE = "https://openlibrary.org" + +REQUIRED_KEYS = ("ol_key", "task_type", "proposed_addition", "comment") + +USER_AGENT = "AgenticCommonsBot/0.1 (wiki-bot@agentic-commons.org)" + + +def load_proposal(path, item_index): + with open(path) as f: + doc = json.load(f) + if isinstance(doc, dict) and "items" in doc: + items = doc["items"] + if not items: + raise ValueError(f"proposal {path}: 'items' is empty") + if item_index < 0 or item_index >= len(items): + raise ValueError(f"--item-index {item_index} out of range (have {len(items)})") + proposal = items[item_index] + elif isinstance(doc, dict) and "ol_key" in doc: + proposal = doc + else: + raise ValueError(f"proposal {path}: expected single proposal or {{items:[...]}}") + for k in REQUIRED_KEYS: + if k not in proposal: + raise ValueError(f"proposal missing required field: {k!r}") + if proposal["task_type"] != "add_alternate_name": + raise ValueError( + f"only task_type='add_alternate_name' is supported in this MVP " + f"(got {proposal['task_type']!r})" + ) + if not proposal["ol_key"].startswith("/authors/"): + raise ValueError( + f"alternate_name task only supports /authors/* pages " + f"(got {proposal['ol_key']!r})" + ) + return proposal + + +def make_opener(user_agent): + jar = http.cookiejar.CookieJar() + opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(jar), + urllib.request.HTTPRedirectHandler(), + ) + opener.addheaders = [ + ("User-Agent", user_agent), + ("Accept", "application/json"), + ] + return opener, jar + + +def login_s3(opener, access, secret): + """Form-POST to /account/login with S3 access+secret. + + This grants a full-privilege session (writes allowed). The alternative — + email+password — yields an anti-bot-restricted session (PUT -> 403). + Matches the auth pattern used by openlibrary-client (olclient). + + On success: HTTP 303 redirect + session cookie set. + """ + # Minimal field set: adding 'remember'/'test' triggers HTTP 500 on OL. + body = urllib.parse.urlencode({ + "access": access, + "secret": secret, + }).encode("utf-8") + req = urllib.request.Request( + f"{BASE}/account/login", + data=body, method="POST", + ) + req.add_header("Content-Type", "application/x-www-form-urlencoded") + with opener.open(req, timeout=30) as r: + body_text = r.read().decode("utf-8", errors="replace") + return r.status, r.geturl(), body_text + + +def fetch_author(opener, ol_key): + """GET /authors/OL...A.json — returns parsed JSON dict.""" + url = f"{BASE}{ol_key}.json" + req = urllib.request.Request(url) + with opener.open(req, timeout=15) as r: + return json.loads(r.read().decode("utf-8")) + + +def put_author(opener, ol_key, author_data): + """PUT /authors/OL...A.json with full updated JSON. + + Returns (status, final_url, body_text). + """ + url = f"{BASE}{ol_key}.json" + body = json.dumps(author_data, ensure_ascii=False).encode("utf-8") + req = urllib.request.Request(url, data=body, method="PUT") + req.add_header("Content-Type", "application/json") + req.add_header("Accept", "application/json") + with opener.open(req, timeout=30) as r: + return r.status, r.geturl(), r.read().decode("utf-8", errors="replace") + + +def main(): + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, + ) + ap.add_argument("--proposal", required=True, help="Path to proposal JSON") + ap.add_argument("--item-index", type=int, default=0) + ap.add_argument("--live", action="store_true", + help="Actually PUT. Default is DRY-RUN (does not write).") + args = ap.parse_args() + + username = os.environ.get("OL_BOT_USERNAME", "AgenticCommonsBot") + access = os.environ.get("OL_BOT_ACCESS") + secret = os.environ.get("OL_BOT_SECRET") + if not access or not secret: + print("ERROR: OL_BOT_ACCESS and OL_BOT_SECRET env vars are required.", file=sys.stderr) + print(" Get them from https://archive.org/account/s3.php (Internet", file=sys.stderr) + print(" Archive shares OL's account system).", file=sys.stderr) + sys.exit(2) + + proposal = load_proposal(args.proposal, args.item_index) + ol_key = proposal["ol_key"] + addition = proposal["proposed_addition"] + comment = proposal["comment"] + + print("=" * 72, file=sys.stderr) + print(f"OL alternate-name bot — {'LIVE' if args.live else 'DRY-RUN'}", file=sys.stderr) + print(f"Proposal: {args.proposal} (item {args.item_index})", file=sys.stderr) + print(f"OL key: {ol_key}", file=sys.stderr) + print(f"URL: {BASE}{ol_key}", file=sys.stderr) + print(f"User: {username} (login via S3 access key {access[:6]}…)", file=sys.stderr) + print(f"Adding: {addition!r}", file=sys.stderr) + print("=" * 72, file=sys.stderr) + + opener, jar = make_opener(USER_AGENT) + + print("\n[1] POST /account/login (S3 auth) …", file=sys.stderr) + try: + status, final_url, body = login_s3(opener, access, secret) + print(f" HTTP {status}, landed at {final_url}", file=sys.stderr) + cookies = [(c.name, c.value[:10] + "…") for c in jar] + print(f" cookies set: {cookies}", file=sys.stderr) + except Exception as e: + print(f" login failed: {type(e).__name__}: {e}", file=sys.stderr) + sys.exit(3) + + print(f"\n[2] GET {ol_key}.json (current author data) …", file=sys.stderr) + try: + author = fetch_author(opener, ol_key) + except Exception as e: + print(f" fetch failed: {type(e).__name__}: {e}", file=sys.stderr) + sys.exit(4) + + current_names = author.get("alternate_names") or [] + print(f" current alternate_names ({len(current_names)}): {current_names}", file=sys.stderr) + + if addition in current_names: + print(f"\n⚠️ {addition!r} already in alternate_names — skipping", file=sys.stderr) + return + + # Construct the updated author payload (do not mutate other fields) + updated = dict(author) + updated["alternate_names"] = current_names + [addition] + updated["_comment"] = comment + + print(f"\n[3] Prepared PUT payload:", file=sys.stderr) + print(f" alternate_names: {updated['alternate_names']}", file=sys.stderr) + print(f" _comment (first 120 chars): {comment[:120]!r}", file=sys.stderr) + + if not args.live: + print(f"\n[DRY-RUN] Would PUT to {BASE}{ol_key}.json", file=sys.stderr) + print("[DRY-RUN] Re-run with --live to actually submit.", file=sys.stderr) + return + + print(f"\n[4] LIVE PUT → {BASE}{ol_key}.json", file=sys.stderr) + try: + status, result_url, body = put_author(opener, ol_key, updated) + print(f" HTTP {status}", file=sys.stderr) + print(f" result url: {result_url}", file=sys.stderr) + print(f" body[:300]: {body[:300]!r}", file=sys.stderr) + except urllib.error.HTTPError as e: + print(f" HTTP {e.code}: {e.reason}", file=sys.stderr) + print(f" body[:500]: {e.read()[:500].decode('utf-8', errors='replace')!r}", file=sys.stderr) + sys.exit(5) + + print(f"\n[5] Verify via GET {ol_key}.json …", file=sys.stderr) + fresh = fetch_author(opener, ol_key) + final_names = fresh.get("alternate_names") or [] + if addition in final_names: + print(f"\n✅ SUCCESS — {addition!r} now in alternate_names", file=sys.stderr) + print(f" final list ({len(final_names)}): {final_names}", file=sys.stderr) + print(f" see: {BASE}{ol_key}", file=sys.stderr) + else: + print(f"\n❌ NOT YET present in alternate_names — may need cache refresh", file=sys.stderr) + print(f" observed list: {final_names}", file=sys.stderr) + + +if __name__ == "__main__": + main() From edccd5d3ff6ae25a4835a8876ab20f4c35273959 Mon Sep 17 00:00:00 2001 From: Wendy Date: Thu, 11 Jun 2026 06:18:43 +0000 Subject: [PATCH 2/2] Redesign as deterministic Wikidata-to-OL sync (no LLM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback on internetarchive/openlibrary#12887 (@tfmorris): - Drop LLM-driven proposal flow entirely; bot is now a simple GET-Wikidata → diff → PUT-OL pipeline. - Discovery: stream the monthly author dump, filter to records that have remote_ids.wikidata and lack any non-Latin alternate_names. No more crawling OL search APIs. - Identity match: anchored on the existing OL↔Wikidata cross-link (Q-id in remote_ids), not on heuristic name matching. - Per-author edit: adds ALL missing non-Latin labels at once (deduped against current alternate_names with NFC normalization, value-level deduped across language codes). - Wikidata is the single authoritative source. Two CLI modes: `discover` (stream dump, emit candidate OL keys) and `sync OL...A` (dry-run by default, --live to PUT). Sample run captured against the canonical Su Tong record (/authors/OL713582A): 41 Wikidata labels → 19 non-Latin → 7 net new after dedup against OL's existing 6 alternate_names. Files: - wikidata_author_alias_bot.py: rewritten, two subcommands - README.md: rewritten for new design - sample_run.txt: captured dry-run output (replaces sample_proposal.json and sample_dry_run.txt from the LLM-driven version) --- AgenticCommonsBot/README.md | 123 ++--- AgenticCommonsBot/sample_dry_run.txt | 22 - AgenticCommonsBot/sample_proposal.json | 25 - AgenticCommonsBot/sample_run.txt | 76 +++ .../wikidata_author_alias_bot.py | 458 +++++++++++------- 5 files changed, 428 insertions(+), 276 deletions(-) delete mode 100644 AgenticCommonsBot/sample_dry_run.txt delete mode 100644 AgenticCommonsBot/sample_proposal.json create mode 100644 AgenticCommonsBot/sample_run.txt diff --git a/AgenticCommonsBot/README.md b/AgenticCommonsBot/README.md index db10b6e7..62fa7b07 100644 --- a/AgenticCommonsBot/README.md +++ b/AgenticCommonsBot/README.md @@ -1,45 +1,72 @@ # AgenticCommonsBot -Adds one well-evidenced entry to an Open Library author's `alternate_names` array, citing Wikidata + a Wikipedia article in another language as independent sources. +Deterministic Wikidata-to-OpenLibrary sync for author `alternate_names`. -Closes #450. +For OpenLibrary authors that already have a cross-linked Wikidata Q-id but +lack one or more non-Latin labels Wikidata has, this bot fetches the +Wikidata labels, diffs against OpenLibrary's current `alternate_names`, +and PUTs the missing non-Latin forms in a single edit. + +Closes [internetarchive/openlibrary#12887](https://github.com/internetarchive/openlibrary/issues/12887). ## What this bot does -For a single OL author at a time: +Two modes, both pure stdlib: -1. GET `/authors/OL...A.json` -2. Verify `alternate_names` is still empty (skip if not — someone else already handled it) -3. Append exactly one proposed addition to `alternate_names` -4. PUT the updated author JSON with an edit comment citing the two evidence URLs +### `discover` — find candidates -That's it. One author, one addition, one PUT. +Stream the monthly author dump (`ol_dump_authors_latest.txt.gz`), filter to +authors that: +- have `remote_ids.wikidata` populated (cross-linked Q-id, anchors identity) +- have no non-Latin form in `alternate_names` (Unicode-block check) -## What this bot does NOT do +Emit OL author keys to stdout, one per line. -- Does not generate the alternate name itself. It reads a pre-built proposal JSON. The proposal upstream is produced by a research worker (which may use an LLM) and then validated by an independent QA gate that re-fetches the two evidence URLs and checks they actually contain the proposed value. -- Does not edit works, editions, subjects, or any field other than `alternate_names`. -- Does not remove, reorder, or replace existing `alternate_names` entries. -- Does not bulk-process. Each invocation handles a single proposal item. +### `sync` — actually edit one author -## Evidence requirement +For one OL author key: +1. POST `/account/login` with S3 keys → write-capable session +2. GET `/authors/.json` → current `alternate_names`, current Q-id link +3. GET Wikidata `Special:EntityData/.json` → all label translations +4. Diff: non-Latin Wikidata labels MINUS OL's current alternate_names (NFC-normalized, value-level dedup across language codes) +5. If diff is empty → skip; otherwise PUT updated record +6. Verify via re-fetch GET -Every proposal must include **at least two independent reputable sources** in the `evidence` array. The default pair: +## What this bot does NOT do + +- Does not use an LLM, fuzzy matching, or any model judgment. Identity match is anchored on the OL↔Wikidata cross-link; missing-form detection is set difference over Unicode-block-filtered labels. +- Does not crawl OL. Candidate discovery uses the monthly bulk dump (one HTTP GET per month). Per-author work uses one GET + one PUT + one verify GET. +- Does not edit works, editions, subjects, or any field other than `alternate_names`. +- Does not remove, reorder, or replace existing `alternate_names` entries — it appends only. +- Does not handle author record duplicates (a separate concern; out of scope here). +- Does not invent or transliterate names. Only labels already present on Wikidata are propagated. -- **Wikidata** — capture the Q-id and the matching multilingual label -- **Wikipedia (other-language article)** — capture the article title in the script being added +## Why this design -Other acceptable corroborators: official publisher page, university faculty page, national library authority record (LoC, BnF, NDL, NLI, etc.). +Following review feedback on issue #12887: -The bot is strict about the two-source minimum. If a proposal arrives with fewer than two evidence entries, the upstream gate rejects it before it reaches the bot. +| Concern | Resolution | +|---|---| +| OL is hammered by crawler traffic | Use monthly bulk dump for candidate discovery, not OL search/list APIs | +| Weak identity matching (matching by name alone) | Require OL record to have `remote_ids.wikidata` populated; identity comes from the existing cross-link, not from heuristics | +| One-addition-per-edit is too conservative | Each edit adds **all** missing non-Latin forms from Wikidata at once | +| Wikidata + Wikipedia aren't really independent sources | Wikidata is the sole authoritative source; the Q-id anchors the match | ## Frequency -Targeting **≤ 8 edits per day** total. Each edit issues 1 GET + 1 PUT + 1 verify GET, paced at 1.5s between requests. Well below polite-bot thresholds. +Per-day cap of 8 edits initially. Each edit makes 1 GET (Wikidata) + 1 GET +(OL) + 1 PUT (OL) + 1 verify GET (OL) = 4 HTTP calls. Monthly dump fetch is +one streamed download (no local persistence). + +`work_count`-descending candidate ordering is a future enhancement — it +requires either an additional OL works-dump pass (~several GB) or batch +queries to OL search, neither of which is in this version. ## Authentication -Uses Internet Archive S3 keys (access + secret) per OL's standard write-API auth path. Get them from https://archive.org/account/s3.php while signed in to the bot account. +Internet Archive S3 keys (access + secret) per OL's standard write-API auth +path. Get them from https://archive.org/account/s3.php while signed in to +the bot account. ```bash export OL_BOT_ACCESS= @@ -48,54 +75,28 @@ export OL_BOT_SECRET= The bot account is `agenticcommonsbot`. -## How to use +## Usage ```bash -# Dry-run (default — does not write): -python3 wikidata_author_alias_bot.py --proposal sample_proposal.json - -# Live submission: -python3 wikidata_author_alias_bot.py --proposal sample_proposal.json --live +# Find first 10 candidates from the monthly dump: +python3 wikidata_author_alias_bot.py discover --limit 10 -# If proposal contains multiple items, pick one with --item-index: -python3 wikidata_author_alias_bot.py --proposal multi_item.json --item-index 2 --live -``` +# Dry-run a single author (no write): +python3 wikidata_author_alias_bot.py sync /authors/OL713582A -## Proposal JSON shape - -See `sample_proposal.json` in this directory for a real example. The shape is: - -```json -{ - "items": [ - { - "ol_key": "/authors/OL2630047A", - "task_type": "add_alternate_name", - "author_name_primary": "Su Tong", - "current_alternate_names": [], - "proposed_addition": "苏童", - "comment": "Adding native-script form per Wikidata Q778276 (zh label '苏童') and Chinese Wikipedia article title '苏童'.", - "evidence": [ - {"source": "Wikidata Q778276", "url": "https://www.wikidata.org/wiki/Q778276", "value": "苏童"}, - {"source": "Wikipedia (zh)", "url": "https://zh.wikipedia.org/wiki/%E8%8B%8F%E7%AB%A5", "value": "苏童"} - ], - "rationale": "Su Tong is a major contemporary Chinese novelist (Wives and Concubines / Raise the Red Lantern). Both Wikidata and zhwiki confirm 苏童 as the canonical native-script form." - } - ] -} +# Actually submit: +python3 wikidata_author_alias_bot.py sync /authors/OL713582A --live ``` -## Edit comment format - -Pure factual citation, with the Wikidata Q-id and Wikipedia article inline. Example: - -> Adding native-script form per Wikidata Q778276 (zh label '苏童') and Chinese Wikipedia article title '苏童'. - -No project attribution, no slogans. +## What a sync run looks like -## What a dry-run looks like +See [`sample_run.txt`](sample_run.txt) for a captured dry-run against the +canonical Su Tong record (`/authors/OL713582A`). Summary of that run: -See `sample_dry_run.txt` in this directory for captured output from a real dry-run against `/authors/OL2630047A` (real OL author, real network calls, no PUT issued). +- 41 Wikidata labels for Q778276 +- 19 non-Latin after Unicode-block filter +- 7 net new after dedup against OL's existing 6 `alternate_names` +- Edit would add: Hebrew, Russian, Thai, Arabic + Egyptian Arabic variant, Korean, Assamese (Bulgarian "Су Тун" was deduped against Russian's identical spelling) ## Maintainer diff --git a/AgenticCommonsBot/sample_dry_run.txt b/AgenticCommonsBot/sample_dry_run.txt deleted file mode 100644 index 48bbf26b..00000000 --- a/AgenticCommonsBot/sample_dry_run.txt +++ /dev/null @@ -1,22 +0,0 @@ -======================================================================== -OL alternate-name bot — DRY-RUN -Proposal: sample_proposal.json (item 0) -OL key: /authors/OL2630047A -URL: https://openlibrary.org/authors/OL2630047A -User: AgenticCommonsBot (login via S3 access key xxxxxx…) -Adding: '苏童' -======================================================================== - -[1] POST /account/login (S3 auth) … - HTTP 200, landed at https://openlibrary.org/ - cookies set: [('session', '/people/ag…'), ('pd', '…'), ('sfw', '…')] - -[2] GET /authors/OL2630047A.json (current author data) … - current alternate_names (0): [] - -[3] Prepared PUT payload: - alternate_names: ['苏童'] - _comment (first 120 chars): "Adding native-script form per Wikidata Q778276 (zh label '苏童') and Chinese Wikipedia article title '苏童'." - -[DRY-RUN] Would PUT to https://openlibrary.org/authors/OL2630047A.json -[DRY-RUN] Re-run with --live to actually submit. diff --git a/AgenticCommonsBot/sample_proposal.json b/AgenticCommonsBot/sample_proposal.json deleted file mode 100644 index 62ddbc6c..00000000 --- a/AgenticCommonsBot/sample_proposal.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "items": [ - { - "ol_key": "/authors/OL2630047A", - "task_type": "add_alternate_name", - "author_name_primary": "Su Tong", - "current_alternate_names": [], - "proposed_addition": "苏童", - "comment": "Adding native-script form per Wikidata Q778276 (zh label '苏童') and Chinese Wikipedia article title '苏童'.", - "evidence": [ - { - "source": "Wikidata Q778276", - "url": "https://www.wikidata.org/wiki/Q778276", - "value": "苏童" - }, - { - "source": "Wikipedia (zh)", - "url": "https://zh.wikipedia.org/wiki/%E8%8B%8F%E7%AB%A5", - "value": "苏童" - } - ], - "rationale": "Su Tong is a major contemporary Chinese novelist (Wives and Concubines / Raise the Red Lantern). Readers searching in Chinese cannot find him by his romanized name. Both Wikidata and the zh-wiki article confirm the simplified-Chinese form 苏童 as the canonical native-script name." - } - ] -} diff --git a/AgenticCommonsBot/sample_run.txt b/AgenticCommonsBot/sample_run.txt new file mode 100644 index 00000000..11b12c69 --- /dev/null +++ b/AgenticCommonsBot/sample_run.txt @@ -0,0 +1,76 @@ +[1] POST /account/login (S3 auth) + HTTP 200 + +[2] GET /authors/OL713582A.json (current OL state) + name: Su Tong + current alternate_names: ['苏童', '蘇童', 'Sū Tóng', 'Tong Zhonggui', '童忠贵', 'Tong Su'] + wikidata Q-id: Q778276 + +[3] GET Wikidata Q778276 labels + total labels: 41 + +[4] Diff — missing non-Latin labels: 7 + he סו טונג + ru Су Тун + th ซูถง + ar سو تونغ + ko 쑤퉁 + arz سو تونج + as চু টং + +[DRY-RUN] Would PUT /authors/OL713582A.json + edit comment: Added 7 non-Latin label(s) from Wikidata Q778276: 'סו טונג', 'Су Тун', 'ซูถง', 'سو تونغ', '쑤퉁', 'سو تونج', 'চু টং' + new alternate_names (13): ['Sū Tóng', 'Tong Su', 'Tong Zhonggui', 'Су Тун', 'סו טונג', 'سو تونج', 'سو تونغ', 'চু টং', 'ซูถง', '童忠贵', '苏童', '蘇童', '쑤퉁'] + + Re-run with --live to actually submit. +{ + "status": "dry_run", + "ol_key": "/authors/OL713582A", + "qid": "Q778276", + "to_add": [ + [ + "he", + "סו טונג" + ], + [ + "ru", + "Су Тун" + ], + [ + "th", + "ซูถง" + ], + [ + "ar", + "سو تونغ" + ], + [ + "ko", + "쑤퉁" + ], + [ + "arz", + "سو تونج" + ], + [ + "as", + "চু টং" + ] + ], + "new_alternate_names": [ + "Sū Tóng", + "Tong Su", + "Tong Zhonggui", + "Су Тун", + "סו טונג", + "سو تونج", + "سو تونغ", + "চু টং", + "ซูถง", + "童忠贵", + "苏童", + "蘇童", + "쑤퉁" + ], + "edit_comment": "Added 7 non-Latin label(s) from Wikidata Q778276: 'סו טונג', 'Су Тун', 'ซูถง', 'سو تونغ', '쑤퉁', 'سو تونج', 'চু টং'" +} diff --git a/AgenticCommonsBot/wikidata_author_alias_bot.py b/AgenticCommonsBot/wikidata_author_alias_bot.py index 2a621404..b1d00446 100644 --- a/AgenticCommonsBot/wikidata_author_alias_bot.py +++ b/AgenticCommonsBot/wikidata_author_alias_bot.py @@ -1,82 +1,89 @@ #!/usr/bin/env python3 """ -OpenLibrary alternate-name submission bot. +Wikidata-to-OL author alternate_names sync bot. -Reads a proposal JSON (produced by an upstream research worker and validated -by an independent QA gate) and adds one well-evidenced name variant to an -author's ``alternate_names`` array on OpenLibrary. +Deterministic (no LLM) sync. For OL authors that already have a cross-linked +Wikidata Q-id but are missing one or more non-Latin labels Wikidata has, this +bot fetches the Wikidata labels, diffs against OL's current alternate_names, +and PUTs the missing non-Latin forms in a single edit. -See README.md for the upstream pipeline, evidence requirements, and rate-limit -posture. +Two modes: -Required env vars: - OL_BOT_ACCESS — S3-style access key from https://archive.org/account/s3.php - OL_BOT_SECRET — S3-style secret key from same page - OL_BOT_USERNAME — display username (only used in logs); defaults to - "AgenticCommonsBot" + discover Stream the OL monthly author dump and emit candidate OL keys to + stdout. Filters to authors that (a) have remote_ids.wikidata + populated and (b) have no non-Latin form in alternate_names. -Why S3 keys (not email/password): OL grants two different session privilege -levels — email+password sessions are anti-bot-restricted (PUT returns 403); -access+secret sessions get full API write privileges. The S3 keys live on -Internet Archive (https://archive.org/account/s3.php) because OL is an IA -sub-project sharing accounts. + sync For one OL author key, GET the current OL record + the Wikidata + entity, compute the diff, and PUT the missing non-Latin forms. + Defaults to dry-run; pass --live to actually submit. + +Pure stdlib. Auth via Internet Archive S3 keys (S3 access + secret from +https://archive.org/account/s3.php — OL is an IA sub-project that shares +the account system). + +Required env vars (sync mode only): + OL_BOT_ACCESS S3-style access key + OL_BOT_SECRET S3-style secret key Usage: - python3 wikidata_author_alias_bot.py --proposal sample_proposal.json # dry-run - python3 wikidata_author_alias_bot.py --proposal sample_proposal.json --live # submit + # find candidates from monthly dump: + python3 wikidata_author_alias_bot.py discover --limit 10 -Proposal JSON shape: see README.md and sample_proposal.json. + # dry-run a single author: + python3 wikidata_author_alias_bot.py sync /authors/OL713582A -OL auth: POST /account/login with form-encoded access + secret; -session cookie returned, then GET / PUT page .json with the cookie. + # submit for real: + python3 wikidata_author_alias_bot.py sync /authors/OL713582A --live """ +from __future__ import annotations import argparse +import gzip import http.cookiejar import json import os import sys +import unicodedata +import urllib.error import urllib.parse import urllib.request -BASE = "https://openlibrary.org" - -REQUIRED_KEYS = ("ol_key", "task_type", "proposed_addition", "comment") - -USER_AGENT = "AgenticCommonsBot/0.1 (wiki-bot@agentic-commons.org)" - - -def load_proposal(path, item_index): - with open(path) as f: - doc = json.load(f) - if isinstance(doc, dict) and "items" in doc: - items = doc["items"] - if not items: - raise ValueError(f"proposal {path}: 'items' is empty") - if item_index < 0 or item_index >= len(items): - raise ValueError(f"--item-index {item_index} out of range (have {len(items)})") - proposal = items[item_index] - elif isinstance(doc, dict) and "ol_key" in doc: - proposal = doc - else: - raise ValueError(f"proposal {path}: expected single proposal or {{items:[...]}}") - for k in REQUIRED_KEYS: - if k not in proposal: - raise ValueError(f"proposal missing required field: {k!r}") - if proposal["task_type"] != "add_alternate_name": - raise ValueError( - f"only task_type='add_alternate_name' is supported in this MVP " - f"(got {proposal['task_type']!r})" - ) - if not proposal["ol_key"].startswith("/authors/"): - raise ValueError( - f"alternate_name task only supports /authors/* pages " - f"(got {proposal['ol_key']!r})" - ) - return proposal - - -def make_opener(user_agent): +OL_BASE = "https://openlibrary.org" +OL_DUMP_URL = "https://openlibrary.org/data/ol_dump_authors_latest.txt.gz" +WD_BASE = "https://www.wikidata.org" +UA_DEFAULT = "AgenticCommonsBot/0.1 (wiki-bot@agentic-commons.org)" + +# Unicode block markers used to classify a label as "non-Latin". A label is +# non-Latin if any character's Unicode name contains one of these tokens. +NON_LATIN_MARKERS = ( + "CJK", "HIRAGANA", "KATAKANA", "HANGUL", + "ARABIC", "HEBREW", "CYRILLIC", "GREEK", + "DEVANAGARI", "BENGALI", "TAMIL", "THAI", + "TELUGU", "GUJARATI", "KANNADA", "MALAYALAM", + "MYANMAR", "GEORGIAN", "ARMENIAN", "ETHIOPIC", + "TIBETAN", "KHMER", "LAO", "SINHALA", +) + + +def is_non_latin(s: str) -> bool: + for ch in s: + try: + name = unicodedata.name(ch) + except ValueError: + continue + if any(m in name for m in NON_LATIN_MARKERS): + return True + return False + + +def nfc(s: str) -> str: + return unicodedata.normalize("NFC", s.strip()) + + +# ── HTTP helpers ───────────────────────────────────────────────────── + + +def make_opener(user_agent: str): jar = http.cookiejar.CookieJar() opener = urllib.request.build_opener( urllib.request.HTTPCookieProcessor(jar), @@ -89,45 +96,29 @@ def make_opener(user_agent): return opener, jar -def login_s3(opener, access, secret): - """Form-POST to /account/login with S3 access+secret. - - This grants a full-privilege session (writes allowed). The alternative — - email+password — yields an anti-bot-restricted session (PUT -> 403). - Matches the auth pattern used by openlibrary-client (olclient). +def login_s3(opener, access: str, secret: str) -> int: + """POST /account/login with S3 keys → establishes a write-capable session. - On success: HTTP 303 redirect + session cookie set. + Minimal field set: adding remember/test fields triggers HTTP 500. + Matches the auth pattern used by olclient. """ - # Minimal field set: adding 'remember'/'test' triggers HTTP 500 on OL. - body = urllib.parse.urlencode({ - "access": access, - "secret": secret, - }).encode("utf-8") - req = urllib.request.Request( - f"{BASE}/account/login", - data=body, method="POST", - ) + body = urllib.parse.urlencode({"access": access, "secret": secret}).encode() + req = urllib.request.Request(f"{OL_BASE}/account/login", data=body, method="POST") req.add_header("Content-Type", "application/x-www-form-urlencoded") with opener.open(req, timeout=30) as r: - body_text = r.read().decode("utf-8", errors="replace") - return r.status, r.geturl(), body_text + return r.status -def fetch_author(opener, ol_key): - """GET /authors/OL...A.json — returns parsed JSON dict.""" - url = f"{BASE}{ol_key}.json" +def fetch_ol_author(opener, ol_key: str) -> dict: + url = f"{OL_BASE}{ol_key}.json" req = urllib.request.Request(url) with opener.open(req, timeout=15) as r: return json.loads(r.read().decode("utf-8")) -def put_author(opener, ol_key, author_data): - """PUT /authors/OL...A.json with full updated JSON. - - Returns (status, final_url, body_text). - """ - url = f"{BASE}{ol_key}.json" - body = json.dumps(author_data, ensure_ascii=False).encode("utf-8") +def put_ol_author(opener, ol_key: str, data: dict): + url = f"{OL_BASE}{ol_key}.json" + body = json.dumps(data, ensure_ascii=False).encode("utf-8") req = urllib.request.Request(url, data=body, method="PUT") req.add_header("Content-Type", "application/json") req.add_header("Accept", "application/json") @@ -135,100 +126,231 @@ def put_author(opener, ol_key, author_data): return r.status, r.geturl(), r.read().decode("utf-8", errors="replace") -def main(): - ap = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, +def fetch_wikidata_labels(qid: str, user_agent: str) -> dict: + """Return {language_code: label} for the Wikidata entity.""" + url = f"{WD_BASE}/wiki/Special:EntityData/{qid}.json" + req = urllib.request.Request(url, headers={"User-Agent": user_agent}) + with urllib.request.urlopen(req, timeout=30) as r: + data = json.loads(r.read().decode("utf-8")) + ent = data["entities"].get(qid, {}) + labels = ent.get("labels", {}) + return {code: l["value"] for code, l in labels.items()} + + +# ── discover mode ──────────────────────────────────────────────────── + + +def discover_candidates(dump_url: str = OL_DUMP_URL, + user_agent: str = UA_DEFAULT, + limit: int | None = None): + """Stream the OL author dump, yield (ol_key, record) for candidates. + + A candidate is an author that: + - has `remote_ids.wikidata` set (the cross-link anchor) + - has no non-Latin form in `alternate_names` + """ + req = urllib.request.Request(dump_url, headers={"User-Agent": user_agent}) + yielded = 0 + scanned = 0 + skipped_no_wikidata = 0 + skipped_already_has_non_latin = 0 + + with urllib.request.urlopen(req, timeout=120) as resp: + with gzip.GzipFile(fileobj=resp) as gz: + for raw in gz: + scanned += 1 + if limit and yielded >= limit: + break + parts = raw.decode("utf-8", errors="replace").rstrip("\n").split("\t") + if len(parts) != 5: + continue + if parts[0] != "/type/author": + continue + try: + rec = json.loads(parts[4]) + except json.JSONDecodeError: + continue + qid = (rec.get("remote_ids") or {}).get("wikidata") + if not qid: + skipped_no_wikidata += 1 + continue + alts = rec.get("alternate_names") or [] + if any(is_non_latin(a) for a in alts): + skipped_already_has_non_latin += 1 + continue + yielded += 1 + yield rec["key"], rec + + print( + f"# discover summary: scanned={scanned} yielded={yielded} " + f"skipped_no_wikidata={skipped_no_wikidata} " + f"skipped_already_has_non_latin={skipped_already_has_non_latin}", + file=sys.stderr, ) - ap.add_argument("--proposal", required=True, help="Path to proposal JSON") - ap.add_argument("--item-index", type=int, default=0) - ap.add_argument("--live", action="store_true", - help="Actually PUT. Default is DRY-RUN (does not write).") - args = ap.parse_args() - username = os.environ.get("OL_BOT_USERNAME", "AgenticCommonsBot") - access = os.environ.get("OL_BOT_ACCESS") - secret = os.environ.get("OL_BOT_SECRET") - if not access or not secret: - print("ERROR: OL_BOT_ACCESS and OL_BOT_SECRET env vars are required.", file=sys.stderr) - print(" Get them from https://archive.org/account/s3.php (Internet", file=sys.stderr) - print(" Archive shares OL's account system).", file=sys.stderr) - sys.exit(2) - - proposal = load_proposal(args.proposal, args.item_index) - ol_key = proposal["ol_key"] - addition = proposal["proposed_addition"] - comment = proposal["comment"] - - print("=" * 72, file=sys.stderr) - print(f"OL alternate-name bot — {'LIVE' if args.live else 'DRY-RUN'}", file=sys.stderr) - print(f"Proposal: {args.proposal} (item {args.item_index})", file=sys.stderr) - print(f"OL key: {ol_key}", file=sys.stderr) - print(f"URL: {BASE}{ol_key}", file=sys.stderr) - print(f"User: {username} (login via S3 access key {access[:6]}…)", file=sys.stderr) - print(f"Adding: {addition!r}", file=sys.stderr) - print("=" * 72, file=sys.stderr) - - opener, jar = make_opener(USER_AGENT) - - print("\n[1] POST /account/login (S3 auth) …", file=sys.stderr) + +# ── sync mode ──────────────────────────────────────────────────────── + + +def compute_additions(author_record: dict, wd_labels: dict) -> list: + """Return [(lang_code, label)] of non-Latin Wikidata labels NOT in OL. + + Dedup is value-based (NFC normalized) — multiple Wikidata language codes + often share the same string (e.g. zh + zh-cn + zh-hans all = '苏童'), + so we only add the first unique form. + """ + have = {nfc(author_record.get("name") or "")} + for a in (author_record.get("alternate_names") or []): + have.add(nfc(a)) + have.discard("") + + seen = set() + to_add = [] + for code, val in wd_labels.items(): + if not is_non_latin(val): + continue + n = nfc(val) + if n in have or n in seen: + continue + seen.add(n) + to_add.append((code, val)) + return to_add + + +def sync_author(ol_key: str, access: str, secret: str, + user_agent: str = UA_DEFAULT, live: bool = False) -> dict: + """Sync one OL author. Returns a result dict.""" + opener, _jar = make_opener(user_agent) + + print(f"[1] POST /account/login (S3 auth)", file=sys.stderr) try: - status, final_url, body = login_s3(opener, access, secret) - print(f" HTTP {status}, landed at {final_url}", file=sys.stderr) - cookies = [(c.name, c.value[:10] + "…") for c in jar] - print(f" cookies set: {cookies}", file=sys.stderr) + st = login_s3(opener, access, secret) + print(f" HTTP {st}", file=sys.stderr) except Exception as e: - print(f" login failed: {type(e).__name__}: {e}", file=sys.stderr) - sys.exit(3) + return {"status": "login_failed", "error": f"{type(e).__name__}: {e}"} - print(f"\n[2] GET {ol_key}.json (current author data) …", file=sys.stderr) + print(f"\n[2] GET {ol_key}.json (current OL state)", file=sys.stderr) try: - author = fetch_author(opener, ol_key) - except Exception as e: - print(f" fetch failed: {type(e).__name__}: {e}", file=sys.stderr) - sys.exit(4) + author = fetch_ol_author(opener, ol_key) + except urllib.error.HTTPError as e: + return {"status": "fetch_failed", "ol_key": ol_key, "http_code": e.code} + + qid = (author.get("remote_ids") or {}).get("wikidata") + print(f" name: {author.get('name')}", file=sys.stderr) + print(f" current alternate_names: {author.get('alternate_names') or []}", file=sys.stderr) + print(f" wikidata Q-id: {qid or '(none)'}", file=sys.stderr) + + if not qid: + return {"status": "skipped", "reason": "no_wikidata_link", "ol_key": ol_key} + + print(f"\n[3] GET Wikidata {qid} labels", file=sys.stderr) + labels = fetch_wikidata_labels(qid, user_agent) + print(f" total labels: {len(labels)}", file=sys.stderr) + + to_add = compute_additions(author, labels) + print(f"\n[4] Diff — missing non-Latin labels: {len(to_add)}", file=sys.stderr) + for code, val in to_add: + print(f" {code:10} {val}", file=sys.stderr) + + if not to_add: + return { + "status": "skipped", + "reason": "no_missing_non_latin_labels", + "ol_key": ol_key, + "qid": qid, + } + + additions_only = [val for _, val in to_add] + new_alt = sorted(set(author.get("alternate_names") or []) | set(additions_only)) + updated = dict(author) + updated["alternate_names"] = new_alt + updated["_comment"] = ( + f"Added {len(to_add)} non-Latin label(s) from Wikidata {qid}: " + + ", ".join(f"'{v}'" for v in additions_only) + ) + + if not live: + print(f"\n[DRY-RUN] Would PUT {ol_key}.json", file=sys.stderr) + print(f" edit comment: {updated['_comment']}", file=sys.stderr) + print(f" new alternate_names ({len(new_alt)}): {new_alt}", file=sys.stderr) + print(f"\n Re-run with --live to actually submit.", file=sys.stderr) + return { + "status": "dry_run", + "ol_key": ol_key, + "qid": qid, + "to_add": to_add, + "new_alternate_names": new_alt, + "edit_comment": updated["_comment"], + } + + print(f"\n[5] LIVE PUT {ol_key}.json", file=sys.stderr) + try: + st, url, body = put_ol_author(opener, ol_key, updated) + print(f" HTTP {st}", file=sys.stderr) + print(f" body[:300]: {body[:300]!r}", file=sys.stderr) + except urllib.error.HTTPError as e: + body = e.read()[:500].decode("utf-8", errors="replace") + print(f" HTTP {e.code}: {e.reason}", file=sys.stderr) + print(f" body[:500]: {body!r}", file=sys.stderr) + return { + "status": "publish_failed", + "ol_key": ol_key, + "http_code": e.code, + "body": body, + } + + print(f"\n[6] Verify GET {ol_key}.json", file=sys.stderr) + fresh = fetch_ol_author(opener, ol_key) + final = fresh.get("alternate_names") or [] + landed = all(v in final for v in additions_only) + print(f" final alternate_names ({len(final)}): {final}", file=sys.stderr) + print(f" all additions landed: {landed}", file=sys.stderr) + + return { + "status": "published" if landed else "published_unverified", + "ol_key": ol_key, + "qid": qid, + "added": to_add, + "final_alternate_names": final, + "verified": landed, + } + + +# ── main ───────────────────────────────────────────────────────────── - current_names = author.get("alternate_names") or [] - print(f" current alternate_names ({len(current_names)}): {current_names}", file=sys.stderr) - if addition in current_names: - print(f"\n⚠️ {addition!r} already in alternate_names — skipping", file=sys.stderr) - return +def main(): + ap = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = ap.add_subparsers(dest="mode", required=True) - # Construct the updated author payload (do not mutate other fields) - updated = dict(author) - updated["alternate_names"] = current_names + [addition] - updated["_comment"] = comment + p_disc = sub.add_parser("discover", help="Stream OL dump → emit candidate OL keys to stdout") + p_disc.add_argument("--limit", type=int, default=None, help="Stop after N candidates") + p_disc.add_argument("--dump-url", default=OL_DUMP_URL) - print(f"\n[3] Prepared PUT payload:", file=sys.stderr) - print(f" alternate_names: {updated['alternate_names']}", file=sys.stderr) - print(f" _comment (first 120 chars): {comment[:120]!r}", file=sys.stderr) + p_sync = sub.add_parser("sync", help="Sync a single OL author") + p_sync.add_argument("ol_key", help="OL author key, e.g. /authors/OL713582A") + p_sync.add_argument("--live", action="store_true", + help="Actually PUT. Default is DRY-RUN (does not write).") - if not args.live: - print(f"\n[DRY-RUN] Would PUT to {BASE}{ol_key}.json", file=sys.stderr) - print("[DRY-RUN] Re-run with --live to actually submit.", file=sys.stderr) + args = ap.parse_args() + + if args.mode == "discover": + for ol_key, _rec in discover_candidates(dump_url=args.dump_url, limit=args.limit): + print(ol_key) return - print(f"\n[4] LIVE PUT → {BASE}{ol_key}.json", file=sys.stderr) - try: - status, result_url, body = put_author(opener, ol_key, updated) - print(f" HTTP {status}", file=sys.stderr) - print(f" result url: {result_url}", file=sys.stderr) - print(f" body[:300]: {body[:300]!r}", file=sys.stderr) - except urllib.error.HTTPError as e: - print(f" HTTP {e.code}: {e.reason}", file=sys.stderr) - print(f" body[:500]: {e.read()[:500].decode('utf-8', errors='replace')!r}", file=sys.stderr) - sys.exit(5) - - print(f"\n[5] Verify via GET {ol_key}.json …", file=sys.stderr) - fresh = fetch_author(opener, ol_key) - final_names = fresh.get("alternate_names") or [] - if addition in final_names: - print(f"\n✅ SUCCESS — {addition!r} now in alternate_names", file=sys.stderr) - print(f" final list ({len(final_names)}): {final_names}", file=sys.stderr) - print(f" see: {BASE}{ol_key}", file=sys.stderr) - else: - print(f"\n❌ NOT YET present in alternate_names — may need cache refresh", file=sys.stderr) - print(f" observed list: {final_names}", file=sys.stderr) + if args.mode == "sync": + access = os.environ.get("OL_BOT_ACCESS") + secret = os.environ.get("OL_BOT_SECRET") + if not access or not secret: + print("ERROR: OL_BOT_ACCESS and OL_BOT_SECRET env vars are required.", file=sys.stderr) + print(" Get them from https://archive.org/account/s3.php", file=sys.stderr) + sys.exit(2) + result = sync_author(args.ol_key, access, secret, UA_DEFAULT, live=args.live) + print(json.dumps(result, ensure_ascii=False, indent=2)) if __name__ == "__main__":