From 8046ecdbdc66a8495175712e8519c6ebd71065bf Mon Sep 17 00:00:00 2001 From: guanlili Date: Thu, 12 Nov 2026 20:29:57 +0800 Subject: [PATCH] fix: protect issue, project field and onboarding writes --- SKILL.md | 3 +- references/methodology.md | 4 +- scripts/_common.py | 7 +- scripts/fill-field.py | 46 +++++---- scripts/new-project.sh | 69 +++++++------- scripts/reconcile.py | 136 +++++++++++++++------------ tests/test_ledger_write_integrity.py | 63 +++++++++++++ 7 files changed, 211 insertions(+), 117 deletions(-) create mode 100644 tests/test_ledger_write_integrity.py diff --git a/SKILL.md b/SKILL.md index c23977c..4823dd2 100644 --- a/SKILL.md +++ b/SKILL.md @@ -65,7 +65,8 @@ for fv in "类型:需求" "来源:甲方" "需求人:张三" "工时(h):8" "开 ## 场景 C:查看与导出(休眠扩展) ```bash -python3 scripts/reconcile.py --project --repo --dry-run # C1 进度+体检 +python3 scripts/reconcile.py --project --repo --data <账本.json> --dry-run # C1 对账体检 +# 启用交付轮次的仓库可用 --from-labels 替代 --data,仅核对轮次字段 python3 scripts/export-timesheet.py --project [--round 轮次] [--customer] [-o xlsx] # C2 验收表 python3 scripts/fill-field.py --project --field <字段> --value <值> [--where 来源=甲方] [--dry-run] # C3 批量填字段(读回验证内置,幂等重跑) ``` diff --git a/references/methodology.md b/references/methodology.md index 0459084..c206a46 100644 --- a/references/methodology.md +++ b/references/methodology.md @@ -34,7 +34,7 @@ **对账真值文件(`reconcile.py --data`)格式**: ```json -[{"title": "issue 标题", "fields": {"类型": "需求", "来源": "内部", "工时(h)": "2", "开始": "2026-11-08", "结束": "2026-11-08"}}] +[{"number": 123, "title": "issue 标题", "fields": {"类型": "需求", "来源": "内部", "工时(h)": "2", "开始": "2026-11-08", "结束": "2026-11-08"}}] ``` - 数字字段写 `2` / `2.0` / `"2"` 等价,引擎归一后比较——写法不同不会被判成数据漂移 @@ -60,7 +60,7 @@ 4. 工时统一 AI 评估口径:**人类等价工作量**(熟练工程师完成同样工作所需时长,非 AI 墙钟——2026-11-08 定案,与历史账/飞书记录同口径,AI 压缩的是墙钟不是交付价值),**实时与回填一律从宽、宁多勿漏**(回填仅见 git 结果,过程中的诊断/实测/死胡同不可见,只能量级基准上界),依据写进 issue 工时段;开发期 commit message 末尾带 issue 号 `(#N)` 5. 该仓库 README「交付轮次工时汇总」追加本轮汇总表 6. 新轮次首条 issue 前追加轮次选项并立即 reconcile 重建(见坑 6);轮次以 `轮次:<名>` label 镜像在 issue 上 -7. AI 自动职责(用户无输入):记账完成后顺手 `reconcile.py --dry-run` 自检有漂移即修;**交付轮次收官时自动导出该轮验收表**并告知位置 +7. AI 自动职责(用户无输入):记账完成后顺手 `reconcile.py --data <账本.json> --dry-run` 自检有漂移即修;**交付轮次收官时自动导出该轮验收表**并告知位置 ## 五、验收导出 diff --git a/scripts/_common.py b/scripts/_common.py index cba66db..5c995c2 100644 --- a/scripts/_common.py +++ b/scripts/_common.py @@ -24,9 +24,12 @@ def run(args, retries=5, timeout=180, pace=0): return 1, "", last -def gql(query, retries=5): +def gql(query, retries=5, *, cursor=None): """GraphQL 查询 → 解析后的 data;失败返回 None(内部自动退避)。""" - rc, out, _ = run(["gh", "api", "graphql", "-f", "query=" + query], retries=retries) + args = ["gh", "api", "graphql", "-f", "query=" + query] + if cursor is not None: + args += ["-f", f"cursor={cursor}"] + rc, out, _ = run(args, retries=retries) if rc != 0: return None try: diff --git a/scripts/fill-field.py b/scripts/fill-field.py index a008a2b..dd3fba8 100755 --- a/scripts/fill-field.py +++ b/scripts/fill-field.py @@ -17,7 +17,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _common import gql, norm, run -ITEMS_Q = '''query { viewer { projectV2(number: %d) { items(first: 100) { +ITEMS_Q = '''query($cursor: String) { viewer { projectV2(number: %d) { items(first: 100, after: $cursor) { + pageInfo { hasNextPage endCursor } nodes { content { ... on Issue { number title url } } fieldValues(first: 20) { nodes { @@ -31,21 +32,26 @@ def read_items(project): - data = gql(ITEMS_Q % project) - if not data: - sys.exit("读取 Project 失败(限流?稍后重跑)") - items = [] - for n in data["viewer"]["projectV2"]["items"]["nodes"]: - c = n.get("content") - if not c: - continue - vals = {} - for f in n["fieldValues"]["nodes"]: - fn = (f.get("field") or {}).get("name") - if fn and fn not in vals: - vals[fn] = f.get("text") if "text" in f else (f.get("name") or f.get("number") or f.get("date")) - items.append({"title": c["title"], "url": c["url"], "vals": vals}) - return items + items, cursor = [], None + while True: + data = gql(ITEMS_Q % project, cursor=cursor) + project_data = (data or {}).get("viewer", {}).get("projectV2") + if not project_data: + sys.exit("读取 Project 失败,请检查项目编号和访问权限") + page = project_data["items"] + for node in page["nodes"]: + content = node.get("content") or {} + if not content.get("url"): + continue + vals = {} + for value in node["fieldValues"]["nodes"]: + field = (value.get("field") or {}).get("name") + if field and field not in vals: + vals[field] = value.get("text", value.get("name", value.get("number", value.get("date")))) + items.append({"title": content["title"], "url": content["url"], "vals": vals}) + if not page["pageInfo"]["hasNextPage"]: + return items + cursor = page["pageInfo"]["endCursor"] def plan_changes(items, field, value, where=None): @@ -71,7 +77,10 @@ def main(): ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() - for sweep in range(1, args.sweeps + 1): + if args.sweeps < 1: + ap.error("--sweeps 必须大于 0") + + for sweep in range(1, args.sweeps + 2): todo = plan_changes(read_items(args.project), args.field, args.value, args.where) print(f"[sweep{sweep}] 需修改 {len(todo)} 条", flush=True) if not todo: @@ -81,13 +90,14 @@ def main(): for it in todo[:15]: print(f" DRIFT {it['title'][:36]} | 现={it['vals'].get(args.field)!r} → {args.value!r}") return + if sweep > args.sweeps: + sys.exit(f"FILL INCOMPLETE:读回仍有 {len(todo)} 条差异") for it in todo: rc, _, err = run(["gh", "project", "item-edit", str(args.project), "--owner", "@me", "--url", it["url"], "--field", args.field, "--value", args.value], pace=1.5) if rc != 0: print(f" FAIL {it['title'][:30]}: {err[:60]}", flush=True) time.sleep(10) - print("FILL DONE(达最大轮数,如仍有差异请重跑——幂等只补差异)") if __name__ == "__main__": diff --git a/scripts/new-project.sh b/scripts/new-project.sh index c5be43f..34172f1 100755 --- a/scripts/new-project.sh +++ b/scripts/new-project.sh @@ -184,25 +184,35 @@ def api(path, method="GET", fields=None): import time as _t for attempt in range(4): r = subprocess.run(cmd, capture_output=True, text=True) - if r.returncode == 0 or "404" in r.stderr or "NOT_FOUND" in r.stderr: - return r.returncode, r.stdout, r.stderr # 成功或确定不存在,直接返回 + if r.returncode == 0 or any(f"HTTP {status}" in r.stderr for status in (404, 409, 422)): + return r.returncode, r.stdout, r.stderr _t.sleep(10 * (attempt + 1)) # 网络抖动重试 return r.returncode, r.stdout, r.stderr -def put_file(path_in_repo, content, message): +def read_file(path_in_repo): enc = urllib.parse.quote(path_in_repo) code, out, err = api(f"repos/{repo}/contents/{enc}") - fields = {"message": message, "content": base64.b64encode(content.encode()).decode()} if code == 0: - fields["sha"] = json.loads(out)["sha"] - if content.strip() == base64.b64decode(json.loads(out)["content"]).decode("utf-8", "ignore").strip(): - print(f" 已存在且内容一致,跳过: {path_in_repo}") - return - code, out, err = api(f"repos/{repo}/contents/{enc}", "PUT", fields) + data = json.loads(out) + return base64.b64decode(data["content"]).decode("utf-8"), data["sha"] + if "HTTP 404" in err: + return "", None + raise SystemExit(f"读取失败 {path_in_repo}: {err.strip()}") + + +def put_file(path_in_repo, content, message, old, sha): + if sha and content.strip() == old.strip(): + print(f" 已存在且内容一致,跳过: {path_in_repo}") + return + enc = urllib.parse.quote(path_in_repo) + fields = {"message": message, "content": base64.b64encode(content.encode()).decode()} + if sha: + fields["sha"] = sha + code, _, err = api(f"repos/{repo}/contents/{enc}", "PUT", fields) if code != 0: - print(f" WARN 写入失败 {path_in_repo}: {err.strip()[:200]}") - else: - print(f" 已写入: {path_in_repo}") + raise SystemExit(f"写入失败 {path_in_repo}: {err.strip()}") + print(f" 已写入: {path_in_repo}") + # 小节幂等合并:标记已注入→替换到下一顶级标题(保留其后内容);未注入→追加;返回 None=无需写 def merge_section(old, sec, marker): @@ -218,33 +228,24 @@ def merge_section(old, sec, marker): base = old[:idx].rstrip() if idx >= 0 else old.rstrip() return base + ("\n\n" if base else "") + sec + ("\n" + tail if tail else "\n") -# CLAUDE.md(幂等标记=小节标题;v1.2.1 起模板正文不含 URL,勿用 url 判断——否则重跑重复追加) +# 用同一次读取的内容和 SHA 合并,避免覆盖并发编辑。 marker = "## 项目管理(devledger" -code, out, _ = api(f"repos/{repo}/contents/CLAUDE.md") -if code == 0: - old = base64.b64decode(json.loads(out)["content"]).decode("utf-8", "ignore") - merged = merge_section(old, section, marker) - if merged is None: - print(" CLAUDE.md 小节已是最新,跳过") - else: - put_file("CLAUDE.md", merged, f"chore: 项目管理规范化——CLAUDE.md 记账规范注入/升级(Project #{num})") -else: - put_file("CLAUDE.md", section + "\n", f"chore: 新建 CLAUDE.md——项目管理记账规范(Project #{num})") - -# AGENTS.md(Codex 等读 AGENTS.md 的工具入口;同款幂等合并,文件不存在则带标题新建) amarker = "## 项目管理记账(devledger" -code, out, _ = api(f"repos/{repo}/contents/AGENTS.md") -if code == 0: - old = base64.b64decode(json.loads(out)["content"]).decode("utf-8", "ignore") - merged = merge_section(old, agents_tpl, amarker) +for path, template, section_marker in ( + ("CLAUDE.md", section, marker), + ("AGENTS.md", agents_tpl, amarker), +): + old, sha = read_file(path) + base = "# AGENTS.md\n" if path == "AGENTS.md" and sha is None else old + merged = merge_section(base, template, section_marker) if merged is None: - print(" AGENTS.md 小节已是最新,跳过") + print(f" {path} 小节已是最新,跳过") else: - put_file("AGENTS.md", merged, f"chore: AGENTS.md 记账纪律注入/升级(Project #{num})") -else: - put_file("AGENTS.md", "# AGENTS.md\n\n" + agents_tpl + "\n", f"chore: 新建 AGENTS.md——跨 AI 工具记账纪律(Project #{num})") + put_file(path, merged, f"chore: {path} 记账规范(Project #{num})", old, sha) -put_file(".github/ISSUE_TEMPLATE/需求.md", issue_tpl, f"chore: 项目管理 issue 模板(Project #{num})") +path = ".github/ISSUE_TEMPLATE/需求.md" +old, sha = read_file(path) +put_file(path, issue_tpl, f"chore: 项目管理 issue 模板(Project #{num})", old, sha) PYEOF fi diff --git a/scripts/reconcile.py b/scripts/reconcile.py index 6d02edd..9f25563 100644 --- a/scripts/reconcile.py +++ b/scripts/reconcile.py @@ -6,12 +6,12 @@ python3 reconcile.py --project 4 --repo guanlili/xxx --data data/datang-ledger.json --dry-run # 只看差异 python3 reconcile.py --project N --repo guanlili/xxx --from-labels # 按「轮次:」label 重建轮次字段(坑6善后) -数据文件 data/*.json:[{"title": "...", "fields": {"字段": "值", ...}}, ...](各项目账本真值)。 +数据文件 data/*.json:[{"number": 123, "title": "...", "fields": {"字段": "值", ...}}, ...]。 +number 是 --repo 下的 issue 编号。旧文件省略 number 时,要求仓库内标题唯一。 数字字段(如「工时(h)」)写 2 / 2.0 / "2" 均可:引擎读回后统一归一为字符串再比较。 """ import argparse import json -import subprocess import sys import time @@ -34,7 +34,7 @@ def read_project(number): - """GraphQL 读回全部条目及字段值:{title: {url, vals}}。""" + """GraphQL 读回全部条目及字段值:{url: vals}。""" items, cursor = {}, None while True: args = ["gh", "api", "graphql", "-f", f"query={Q % number}"] @@ -47,11 +47,11 @@ def read_project(number): proj = json.loads(out)["data"]["viewer"]["projectV2"] for n in proj["items"]["nodes"]: c = n.get("content") - if not c: + if not c or not c.get("url"): continue vals = {} for v in n["fieldValues"]["nodes"]: - fn = v.get("field", {}).get("name") + fn = (v.get("field") or {}).get("name") if fn == "Status": vals["Status"] = v.get("name") elif fn in ("类型", "来源", "交付轮次"): @@ -65,93 +65,109 @@ def read_project(number): vals["开始"] = v.get("date") elif fn == "结束": vals["结束"] = v.get("date") - items[c["title"]] = {"url": c["url"], "vals": vals} + items[c["url"]] = vals if not proj["items"]["pageInfo"]["hasNextPage"]: return items cursor = proj["items"]["pageInfo"]["endCursor"] -def issue_title_number_map(repo): - rc, out, _ = run(["gh", "issue", "list", "-R", repo, "--state", "all", "--limit", "300", - "--json", "number,title", "--jq", '.[] | "\\(.number)\\t\\(.title)"']) - m = {} - if rc == 0: - for line in out.splitlines(): - n, _, t = line.partition("\t") - m[t] = n - return m +def read_repo_issues(repo): + rc, out, err = run([ + "gh", "api", f"repos/{repo}/issues?state=all&per_page=100", "--paginate", + "--jq", ".[] | select(has(\"pull_request\") | not) | {number,title,labels:[.labels[].name]}", + ]) + if rc != 0: + sys.exit(f"读取仓库 issue 失败:{err}") + return [json.loads(line) for line in out.splitlines()] -def expect_from_labels(repo): +def expect_from_labels(issues): """按「轮次:<名>」label 生成轮次预期(坑6善后用)。""" - rc, out, _ = run(["gh", "issue", "list", "-R", repo, "--state", "all", "--limit", "300", - "--json", "title,labels", "--jq", '.[] | "\\(.title)\\t\\([.labels[].name] | join(","))"']) expect = [] - if rc == 0: - for line in out.splitlines(): - title, _, labels = line.partition("\t") - rounds = [l[len("轮次:"):] for l in labels.split(",") if l.startswith("轮次:")] - if rounds: - expect.append({"title": title, "fields": {"交付轮次": rounds[0]}}) + for issue in issues: + rounds = [label[len("轮次:"):] for label in issue["labels"] if label.startswith("轮次:")] + if rounds: + expect.append({"number": issue["number"], "title": issue["title"], "fields": {"交付轮次": rounds[0]}}) return expect +def resolve_targets(expect, repo, issues): + by_title = {} + for issue in issues: + by_title.setdefault(issue["title"], []).append(issue["number"]) + targets = {} + for entry in expect: + title = entry.get("title", "") + number = entry.get("number") + if number is None: + matches = by_title.get(title, []) + if len(matches) != 1: + sys.exit(f"标题匹配到 {len(matches)} 条 issue:{title!r},请在数据文件中指定 number") + number = matches[0] + if type(number) is not int or number < 1: + sys.exit(f"无效 issue 编号:{number!r}") + url = f"https://github.com/{repo}/issues/{number}".lower() + if url in targets: + sys.exit(f"数据文件重复指定 issue #{number}") + targets[url] = (f"#{number} {title}".rstrip(), entry["fields"]) + return targets + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--project", type=int, required=True, help="GitHub Project 编号") ap.add_argument("--repo", required=True, help="issue 所在仓库 owner/name") - ap.add_argument("--data", help="账本真值 JSON(与 --from-labels 二选一)") - ap.add_argument("--from-labels", action="store_true", help="按「轮次:」label 重建交付轮次字段") + source = ap.add_mutually_exclusive_group(required=True) + source.add_argument("--data", help="账本真值 JSON(与 --from-labels 二选一)") + source.add_argument("--from-labels", action="store_true", help="按「轮次:」label 重建交付轮次字段") ap.add_argument("--dry-run", action="store_true") args = ap.parse_args() if args.from_labels: - expect = expect_from_labels(args.repo) - print(f"从 label 推得 {len(expect)} 条轮次预期", flush=True) - elif args.data: - expect = json.load(open(args.data)) + expect = expect_from_labels(read_repo_issues(args.repo)) + issues = [] else: - sys.exit("需要 --data 或 --from-labels") - print(f"预期 {len(expect)} 条,dry_run={args.dry_run}", flush=True) + with open(args.data, encoding="utf-8") as data: + expect = json.load(data) + issues = read_repo_issues(args.repo) if any(e.get("number") is None for e in expect) else [] + targets = resolve_targets(expect, args.repo, issues) + print(f"预期 {len(targets)} 条,dry_run={args.dry_run}", flush=True) - for sweep in range(1, 5): + for sweep in range(1, 6): items = read_project(args.project) if items is None: - print(f"[sweep{sweep}] 读失败,5 分钟后重试", flush=True) - time.sleep(300) - continue + sys.exit("RECONCILE INCOMPLETE:无法读回 Project") + current = {url.lower(): vals for url, vals in items.items()} todo = [] - for e in expect: - cur = items.get(e["title"]) - for f, v in e["fields"].items(): - if cur is None or norm(cur["vals"].get(f)) != norm(v): - todo.append((e["title"], cur["url"] if cur else None, f, v)) - print(f"[sweep{sweep}] project {len(items)} 条,需修正 {len(todo)} 处", flush=True) + for url, (label, fields) in targets.items(): + values = current.get(url) + changes = {f: v for f, v in fields.items() if values is None or norm(values.get(f)) != norm(v)} + if changes: + todo.append((url, label, changes)) + drift = sum(len(changes) for _, _, changes in todo) + print(f"[sweep{sweep}] project {len(items)} 条,需修正 {drift} 处", flush=True) if not todo: print("RECONCILE CLEAN", flush=True) return if args.dry_run: - for t, _, f, v in todo[:20]: - print(f" DRIFT {t} {f}={v}") + for _, label, changes in todo: + for field, value in changes.items(): + print(f" DRIFT {label} {field}={value}") return - tmap = issue_title_number_map(args.repo) - for title, url, f, v in todo: - if url is None: - n = tmap.get(title) - if not n: - print(f" 缺 issue 本体:{title}", flush=True) + if sweep == 5: + sys.exit(f"RECONCILE INCOMPLETE:读回仍有 {drift} 处差异") + for url, label, changes in todo: + if url not in current: + rc, _, err = run(["gh", "project", "item-add", str(args.project), "--owner", "@me", "--url", url], pace=2) + if rc != 0: + print(f" FAIL 入板 {label}: {err}", flush=True) continue - url = f"https://github.com/{args.repo}/issues/{n}" - rc, _, e = run(["gh", "project", "item-add", str(args.project), "--owner", "@me", "--url", url]) - print(f" +item-add {title} {'ok' if rc == 0 else e[:60]}", flush=True) - time.sleep(2) - rc, _, e = run(["gh", "project", "item-edit", str(args.project), "--owner", "@me", - "--url", url, "--field", f, "--value", v]) - if rc != 0: - print(f" FAIL {title[:30]} {f}={v}: {e[:60]}", flush=True) - time.sleep(2) + for field, value in changes.items(): + rc, _, err = run(["gh", "project", "item-edit", str(args.project), "--owner", "@me", + "--url", url, "--field", field, "--value", str(value)], pace=2) + if rc != 0: + print(f" FAIL {label} {field}={value}: {err}", flush=True) time.sleep(30) - print("RECONCILE DONE(达最大扫描次数,如仍有差异请重跑)", flush=True) if __name__ == "__main__": diff --git a/tests/test_ledger_write_integrity.py b/tests/test_ledger_write_integrity.py new file mode 100644 index 0000000..c10fb6e --- /dev/null +++ b/tests/test_ledger_write_integrity.py @@ -0,0 +1,63 @@ +"""写入前的定位、分页和仓库文件并发保护。""" +import pathlib +import runpy +import unittest +from unittest.mock import Mock, patch + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +RECONCILE = runpy.run_path(str(SCRIPTS / "reconcile.py")) +FILL = runpy.run_path(str(SCRIPTS / "fill-field.py")) + + +class TestReconcileTargets(unittest.TestCase): + def test_explicit_number_disambiguates_titles(self): + resolve = RECONCILE["resolve_targets"] + issues = [{"title": "同名", "number": 1}, {"title": "同名", "number": 2}] + entry = {"title": "同名", "fields": {"工时(h)": 0}} + with self.assertRaisesRegex(SystemExit, "标题匹配到 2 条"): + resolve([entry], "a/r", issues) + self.assertEqual(list(resolve([dict(entry, number=2)], "a/r", issues)), + ["https://github.com/a/r/issues/2"]) + + +class TestFillPagination(unittest.TestCase): + def test_reads_later_page_and_preserves_zero(self): + read = FILL["read_items"] + + def page(cursor, nodes): + return {"viewer": {"projectV2": {"items": { + "pageInfo": {"hasNextPage": cursor is not None, "endCursor": cursor}, + "nodes": nodes, + }}}} + + node = {"content": {"title": "后一页", "url": "https://github.com/a/r/issues/101"}, + "fieldValues": {"nodes": [{"field": {"name": "工时(h)"}, "number": 0}]}} + with patch.dict(read.__globals__, {"gql": Mock( + side_effect=[page("next", []), page(None, [node])], + )}): + rows = read(1) + self.assertEqual(rows[0]["vals"], {"工时(h)": 0}) + self.assertEqual(read.__globals__["gql"].call_args.kwargs["cursor"], "next") + + +class TestOnboardingWrites(unittest.TestCase): + def setUp(self): + source = (SCRIPTS / "new-project.sh").read_text() + functions = source[source.index("def read_file("):source.index("# 小节幂等合并")] + self.env = {"repo": "a/r"} + exec("import base64, json, urllib.parse\n" + functions, self.env) + + def test_failed_read_does_not_become_creation(self): + self.env["api"] = Mock(return_value=(1, "", "HTTP 503")) + with self.assertRaisesRegex(SystemExit, "读取失败"): + self.env["read_file"]("AGENTS.md") + self.env["api"].assert_called_once() + + def test_conflict_uses_original_sha_and_fails(self): + api = Mock(return_value=(1, "", "HTTP 409")) + self.env["api"] = api + with self.assertRaisesRegex(SystemExit, "写入失败"): + self.env["put_file"]("AGENTS.md", "new", "message", "old", "original-sha") + api.assert_called_once() + self.assertEqual(api.call_args.args[2]["sha"], "original-sha")