From 20972d19004f2085808d433f6c6e5bafb4be4497 Mon Sep 17 00:00:00 2001 From: Caio Lins Date: Tue, 27 Oct 2026 23:57:14 +0900 Subject: [PATCH 1/3] tooling: turn line-capture and line-analyze into skills Move the line-capture and line-analyze commands to .agents/skills, with their scripts next to them, and link .claude/skills to .agents/skills so Claude Code finds them. The capture launcher now runs a dependency preflight first and works on macOS, Linux and Windows (Git Bash). The CDP recorder releases every page-like target so extension windows no longer open frozen. Co-Authored-By: Claude Opus 5.5 --- .agents/commands/line-capture.md | 38 ----- .../line-analyze/SKILL.md} | 52 +++++-- .../scripts}/parse-line-traffic.py | 25 +++- .../scripts}/redact-line-capture.py | 15 +- .agents/skills/line-capture/SKILL.md | 75 ++++++++++ .../scripts/line-capture-preflight.sh | 112 +++++++++++++++ .../line-capture/scripts}/line-cdp-capture.py | 89 ++++++++++-- .../scripts}/line-chrome-capture.sh | 135 +++++++++++++----- .agents/skills/line-implement/REFERENCE.md | 16 +-- .claude/skills | 1 + 10 files changed, 445 insertions(+), 113 deletions(-) delete mode 100644 .agents/commands/line-capture.md rename .agents/{commands/line-analyze.md => skills/line-analyze/SKILL.md} (51%) rename {scripts => .agents/skills/line-analyze/scripts}/parse-line-traffic.py (90%) rename {scripts => .agents/skills/line-analyze/scripts}/redact-line-capture.py (87%) create mode 100644 .agents/skills/line-capture/SKILL.md create mode 100755 .agents/skills/line-capture/scripts/line-capture-preflight.sh rename {scripts => .agents/skills/line-capture/scripts}/line-cdp-capture.py (74%) rename {scripts => .agents/skills/line-capture/scripts}/line-chrome-capture.sh (70%) create mode 120000 .claude/skills diff --git a/.agents/commands/line-capture.md b/.agents/commands/line-capture.md deleted file mode 100644 index b0ecd69c..00000000 --- a/.agents/commands/line-capture.md +++ /dev/null @@ -1,38 +0,0 @@ -Launch Chrome with network capture enabled for LINE Chrome Extension analysis. - -## What to do - -1. **Close any running Chrome instances first** (Chrome only supports one instance per profile with remote debugging). Warn the user if Chrome is running. - -2. Run the capture script: - ``` - bash scripts/line-chrome-capture.sh /tmp/line-chrome-extension-capture - ``` - This launches Chrome with: - - Remote debugging on port 9222 - - Full network logging to `/tmp/line-chrome-extension-capture/line-net-log.json` - - A persistent capture-only Chrome profile at - `~/.cache/line-chrome-extension-capture/chrome-debug-profile` - - Auto-loaded LINE and Codex extensions staged from the user's normal Chrome - profile into `~/.cache/line-chrome-extension-capture/extensions` - - A LINE extension version preflight: - - repo version from `pkg/line/client.go` `ExtensionVersion` - - installed/staged Chrome extension version from `manifest.json` - - a warning if those versions differ - - First-run Chrome prompts pre-skipped: - - "Make Google Chrome the default browser" stays unchecked - - "Automatically send usage statistics and crash reports" stays unchecked - - Chrome/Sync sign-in is skipped; use "Stay Signed Out" if it appears - - If the LINE extension is missing and the user asked for setup, relaunch with - `LINE_CAPTURE_AUTO_LOAD_EXTENSIONS=0`, open the official Web Store listing - for `ophjlpahpchlmihnnnihgmmeilfjmjjc`, verify LINE / LY Corporation, click - Add to Chrome, and accept the standard "Add extension" confirmation - -3. Tell the user: - - Open the LINE Chrome Extension (browser toolbar icon) - - Log in manually if the capture profile is not already logged in - - Perform whatever flow they want to analyze - - When done, quit Chrome completely (Cmd+Q) - -4. Once Chrome is closed, the net-log file is ready. Continue with the - line-analyze workflow. diff --git a/.agents/commands/line-analyze.md b/.agents/skills/line-analyze/SKILL.md similarity index 51% rename from .agents/commands/line-analyze.md rename to .agents/skills/line-analyze/SKILL.md index 98554177..14226037 100644 --- a/.agents/commands/line-analyze.md +++ b/.agents/skills/line-analyze/SKILL.md @@ -1,32 +1,47 @@ -Parse and analyze a LINE Chrome Extension network capture. +--- +name: line-analyze +description: Parses and redacts a LINE Chrome Extension network capture (CDP capture or Chrome net-log) and compares it with the beeper/line bridge code. Use after a line-capture run, or when given a LINE net-log or CDP capture file to analyze. +--- + +# LINE Analyze + +Turns a raw LINE Chrome Extension capture into redacted, readable findings and +compares them with the bridge. Run the commands below from the repository root. ## Input -Use the provided path to a net-log JSON file. If no path is provided, default to +Use the provided path to a capture file. Prefer the CDP capture +(`line-cdp-capture.json`): it includes SSE messages and request bodies. The +Chrome net-log is a backup. If no path is provided, default to +`/tmp/line-chrome-extension-capture/line-cdp-capture.json`, then `/tmp/line-chrome-extension-capture/line-net-log.json`. ## What to do -1. Run the parser: +1. Run the parser. This extracts LINE-related HTTP requests and writes a + `-parsed.json` file: + + ```bash + python3 .agents/skills/line-analyze/scripts/parse-line-traffic.py [] ``` - python3 scripts/parse-line-traffic.py + +2. Redact it before reading or quoting anything. This removes tokens, MIDs, + `picturePath` and raw body bytes: + + ```bash + python3 .agents/skills/line-analyze/scripts/redact-line-capture.py ``` - This extracts LINE-related HTTP requests from the Chrome net-log and writes a `-parsed.json` file. -2. Read the parsed output JSON file. + Read the raw files only locally, and only when the redacted output loses + information you need. Quote or document only redacted snippets. -3. Analyze the traffic and present findings organized by category: +3. Read the redacted output and present findings by category: - **Thrift RPC** (`/api/talk/thrift/`): The main LINE API. Requests use LINE's Thrift-over-HTTP protocol. The `x-lhm` header indicates the Thrift method name. Compare against methods in `pkg/line/methods.go`. - - **Long-poll / SSE** (`/api/talk/long-polling/`, `/api/operation/receive`): Real-time event delivery. LF1 is used during login verification, JQ for login polling, and SSE for ongoing message/operation streaming. Compare against `pkg/line/sse.go` and `pkg/line/client.go`. - - **Auth** (`/api/auth/`): Token refresh and login flows. Compare against login methods in `pkg/line/client.go` (loginV2, confirmE2EELogin, tokenRefresh). - - **Media/OBS** (`obs.line-apps.com`): Media upload/download. Check `x-obs-params` header for operation details, `x-obs-oid` in responses. Compare against `pkg/line/client.go` (UploadOBS, DownloadOBS). - - **Stickers/CDN** (`stickershop`, `scdn`): Sticker image downloads. - - **Secondary API** (`legy-jp.line-apps.com`): Link previews and page info. Compare against `pkg/line/client.go` (GetPageInfo). 4. For each interesting request, explain: @@ -34,9 +49,16 @@ Use the provided path to a net-log JSON file. If no path is provided, default to - The request/response structure - Any headers that carry protocol-specific meaning -5. Highlight any **unimplemented** endpoints or behaviors that differ from what the bridge currently does. Cross-reference with `pkg/line/methods.go` and `pkg/connector/`. +5. Highlight any **unimplemented** endpoints or behaviors that differ from what + the bridge currently does. Cross-reference with `pkg/line/methods.go` and + `pkg/connector/`. -6. After presenting the analysis, ask the user if they want to clean up the capture files (`/tmp/line-chrome-extension-capture/`). Warn that the raw net-log and parsed output contain sensitive data (auth tokens in `X-Line-Access` headers, message IDs, etc.) and should not be left on disk or committed to the repository. If the user agrees, delete the entire capture directory. +6. After presenting the analysis, ask the user if they want to clean up the + capture files (`/tmp/line-chrome-extension-capture/`). Warn that the raw + capture and parsed output contain sensitive data (auth tokens in + `X-Line-Access` headers, message IDs, etc.) and must not be left on disk or + committed to a repository. If the user agrees, delete the entire capture + directory. ## LINE protocol context @@ -44,5 +66,5 @@ Use the provided path to a net-log JSON file. If no path is provided, default to - Thrift RPC uses HTTP POST to `line-chrome-gw.line-apps.com/api/talk/thrift/Talk//`. - Auth token is in `x-line-access` header. - E2EE messages have `chunks` field containing encrypted content (base64 segments). -- Operations have numeric `type` codes — see `pkg/connector/sync.go` and `pkg/connector/handle_message.go` for the operation type switch. +- Operations have numeric `type` codes. See `pkg/connector/sync.go` and `pkg/connector/handle_message.go` for the operation type switch. - Body content in the parsed output may be base64-encoded binary (Thrift serialization). Note the method name from headers rather than trying to decode the binary body. diff --git a/scripts/parse-line-traffic.py b/.agents/skills/line-analyze/scripts/parse-line-traffic.py similarity index 90% rename from scripts/parse-line-traffic.py rename to .agents/skills/line-analyze/scripts/parse-line-traffic.py index dd21a688..bf79df4e 100755 --- a/scripts/parse-line-traffic.py +++ b/.agents/skills/line-analyze/scripts/parse-line-traffic.py @@ -2,7 +2,7 @@ """Parse LINE network captures (CDP or net-log format). Usage: - python3 scripts/parse-line-traffic.py [output.json] + python3 .agents/skills/line-analyze/scripts/parse-line-traffic.py [output.json] Supports two input formats: - CDP capture (from line-cdp-capture.py): has SSE message bodies, Thrift response bodies @@ -231,14 +231,17 @@ def parse_cdp_capture(data: dict) -> dict: entry["status"] = req["status"] # Extract interesting request headers - headers = req.get("headers", {}) + # requestWillBeSentExtraInfo carries the headers Chrome really sent + # (Cookie, Origin, Sec-*, X-Line-*), which requestWillBeSent omits. + headers = {**req.get("headers", {}), **req.get("request_extra_headers", {})} interesting_req = {} for k, v in headers.items(): if k.lower() in ( "content-type", "x-line-access", "x-line-application", "x-lhm", "x-lpv", "x-lsr", "x-obs-params", - "x-line-channeltoken", "accept", - ): + "x-line-channeltoken", "accept", "origin", "user-agent", + "x-line-chrome-version", "x-line-mid", + ) or k.lower().startswith("x-line-") or k.lower().startswith("x-ls"): interesting_req[k] = v if interesting_req: entry["request_headers"] = interesting_req @@ -270,7 +273,19 @@ def parse_cdp_capture(data: dict) -> dict: # Include request body if available if "request_body" in req: - entry["request_body_text"] = req["request_body"] + body = req["request_body"] + if isinstance(body, dict): + # Raw captures keep the bytes in base64; the parsed output only + # keeps the text form and a size + sha256 for binary bodies. + if body.get("error"): + entry["request_body_error"] = body["error"] + else: + entry["request_body_size"] = body.get("size") + entry["request_body_sha256"] = body.get("sha256") + if body.get("text") is not None: + entry["request_body_text"] = body["text"] + else: + entry["request_body_text"] = body entry["timestamp"] = req.get("timestamp", "") line_requests.append(entry) diff --git a/scripts/redact-line-capture.py b/.agents/skills/line-analyze/scripts/redact-line-capture.py similarity index 87% rename from scripts/redact-line-capture.py rename to .agents/skills/line-analyze/scripts/redact-line-capture.py index 8c20f481..fdea3fce 100755 --- a/scripts/redact-line-capture.py +++ b/.agents/skills/line-analyze/scripts/redact-line-capture.py @@ -2,7 +2,7 @@ """Redact sensitive values from LINE capture JSON files. Usage: - python3 scripts/redact-line-capture.py input.json [output.json] + python3 .agents/skills/line-analyze/scripts/redact-line-capture.py input.json [output.json] The default output path is input-redacted.json. This helper preserves JSON structure for protocol analysis while removing credentials, cookies, tokens, @@ -39,6 +39,16 @@ "refresh_token", "session", "certificate", + # Identifiers that point at a person, group or uploaded object. + "mid", + "chatmid", + "contactmid", + "picturepath", + "picturestatus", + "oid", + "x-obs-oid", + # Raw request-body bytes from the CDP recorder (uploads, Thrift payloads). + "base64", } SENSITIVE_KEY_FRAGMENTS = ( @@ -53,7 +63,8 @@ EMAIL_RE = re.compile(r"(?i)[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}") JWT_RE = re.compile(r"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}") -MID_RE = re.compile(r"\b([ucr][0-9a-f]{20,})\b", re.IGNORECASE) +# Not \b: "_" counts as a word character there, which hid MIDs after "_". +MID_RE = re.compile(r"(? -m pip + install --user` for Python packages, using the Python the preflight + reported). Show the user the missing items with the exact commands you plan + to run, ask them to confirm, then install and re-run the preflight until it + exits 0. Never install without that confirmation. The launcher refuses to + start while dependencies are missing. + +2. **Close any running Chrome instances first.** Chrome only supports one + instance per profile with remote debugging. Warn the user if Chrome is + running. + +3. Run the capture script from the checkout: + + ```bash + bash .agents/skills/line-capture/scripts/line-chrome-capture.sh /tmp/line-chrome-extension-capture + ``` + + This launches Chrome with: + - Remote debugging on port 9222 + - Full network logging to `/tmp/line-chrome-extension-capture/line-net-log.json` + - A persistent capture-only Chrome profile at + `~/.cache/line-chrome-extension-capture/chrome-debug-profile` + - Auto-loaded LINE and Codex extensions staged from the user's normal Chrome + profile into `~/.cache/line-chrome-extension-capture/extensions` + - A LINE extension version preflight: + - repo version from `pkg/line/client.go` `ExtensionVersion` + - installed/staged Chrome extension version from `manifest.json` + - a warning if those versions differ + - First-run Chrome prompts pre-skipped: + - "Make Google Chrome the default browser" stays unchecked + - "Automatically send usage statistics and crash reports" stays unchecked + - Chrome/Sync sign-in is skipped; use "Stay Signed Out" if it appears + - If the LINE extension is missing and the user asked for setup, relaunch + with `LINE_CAPTURE_AUTO_LOAD_EXTENSIONS=0`, open the official Web Store + listing for `ophjlpahpchlmihnnnihgmmeilfjmjjc`, verify LINE / LY + Corporation, click Add to Chrome, and accept the standard "Add extension" + confirmation + +4. Tell the user: + - Chrome opens on `about:blank`. Open LINE from the extension's toolbar icon + (puzzle-piece menu if it is not pinned), not by opening the extension's + `index.html` as a tab: that behaves differently from the real popup. + `LINE_CAPTURE_START_URL` overrides the start URL. + - Log in manually if the capture profile is not already logged in + - Perform whatever flow they want to analyze + - Do not open DevTools on the extension: LINE has `debugger;` traps that + pause it, and the recorder does not need DevTools + - Windows the extension opens (Settings, group popups) can start frozen + and blank, with LINE greyed out as "Debugger paused in another tab". The + recorder releases them automatically. If one still freezes, quit Chrome + and run the capture again. + - When done, quit Chrome completely (all windows, not only the tab) + +5. Once Chrome is closed, the capture files are ready. Continue with the + `line-analyze` skill. + +Raw captures are secrets (tokens, cookies, MIDs, message bodies). Never commit +them. See the `line-implement` skill for the full security rules. diff --git a/.agents/skills/line-capture/scripts/line-capture-preflight.sh b/.agents/skills/line-capture/scripts/line-capture-preflight.sh new file mode 100755 index 00000000..ba7aa33c --- /dev/null +++ b/.agents/skills/line-capture/scripts/line-capture-preflight.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Check that everything the LINE Chrome Extension capture needs is installed. +# Works on macOS, Linux and Windows (Git Bash / MSYS2). +# +# Usage: +# .agents/skills/line-capture/scripts/line-capture-preflight.sh # human/agent-readable report +# eval "$(.agents/skills/line-capture/scripts/line-capture-preflight.sh --print-env)" # sets CHROME and the PYTHON array +# +# The report prints the detected OS, one OK line per dependency found and one +# MISSING line per dependency that is not. It deliberately does not say how to +# install anything: the agent running it picks the install method that fits +# the OS and the package managers available, and asks the user first. +# Exits 1 when anything required is missing. +# +# Override detection with CHROME=/path/to/chrome or PYTHON=/path/to/python +# (one executable; spaces in the path are fine). + +set -uo pipefail + +case "$(uname -s)" in + Darwin) OS=macos ;; + Linux) + if grep -qi microsoft /proc/version 2>/dev/null; then OS=wsl; else OS=linux; fi ;; + MINGW* | MSYS* | CYGWIN*) OS=windows ;; + *) OS=unknown ;; +esac + +find_chrome() { + if [ -n "${CHROME:-}" ]; then + [ -x "$CHROME" ] && echo "$CHROME" + return + fi + local candidates=() + case "$OS" in + macos) + candidates=("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" + "$HOME/Applications/Google Chrome.app/Contents/MacOS/Google Chrome") ;; + windows) + candidates=("${PROGRAMFILES:-/c/Program Files}/Google/Chrome/Application/chrome.exe" + "${PROGRAMFILES_X86:-/c/Program Files (x86)}/Google/Chrome/Application/chrome.exe" + "${LOCALAPPDATA:-$HOME/AppData/Local}/Google/Chrome/Application/chrome.exe") ;; + *) + local name + for name in google-chrome google-chrome-stable chromium chromium-browser; do + command -v "$name" >/dev/null 2>&1 && { command -v "$name"; return; } + done ;; + esac + local path + for path in "${candidates[@]}"; do + [ -x "$path" ] && { echo "$path"; return; } + done +} + +# Sets the PYTHON_FOUND array: the executable first, then optional arguments +# ("py -3"). An array keeps a PYTHON override with spaces in its path in one piece. +find_python() { + PYTHON_FOUND=() + local candidate + if [ -n "${PYTHON:-}" ]; then + "$PYTHON" -c "import sys; sys.exit(sys.version_info < (3, 8))" 2>/dev/null && PYTHON_FOUND=("$PYTHON") + return + fi + # "python3" on Windows can be the Microsoft Store stub, so the version check + # also filters out commands that exist but don't run. + for candidate in "python3" "python" "py -3"; do + # shellcheck disable=SC2206 # intentional split of "py -3" + local cmd=($candidate) + if "${cmd[@]}" -c "import sys; sys.exit(sys.version_info < (3, 8))" 2>/dev/null; then + PYTHON_FOUND=("${cmd[@]}") + return + fi + done +} + +CHROME_FOUND="$(find_chrome)" +find_python + +if [ "${1:-}" = "--print-env" ]; then + printf 'CHROME=%q\nPYTHON=(' "$CHROME_FOUND" + [ ${#PYTHON_FOUND[@]} -gt 0 ] && printf '%q ' "${PYTHON_FOUND[@]}" + printf ')\n' + exit 0 +fi + +missing=0 +ok() { echo "OK $1"; } +need() { echo "MISSING $1"; missing=1; } + +echo "OS $OS ($(uname -sm))" +if [ "$OS" = wsl ]; then + echo "NOTE WSL detected: use a Linux Chrome or Chromium inside WSL. Windows chrome.exe cannot use WSL paths or reach the recorder, so run from Git Bash on Windows instead." +fi + +if [ -n "$CHROME_FOUND" ]; then + ok "Google Chrome: $CHROME_FOUND" +else + need "Google Chrome (or Chromium on Linux); set CHROME=/path/to/chrome if it is installed somewhere else" +fi + +if [ ${#PYTHON_FOUND[@]} -gt 0 ]; then + ok "Python 3.8+: ${PYTHON_FOUND[*]} ($("${PYTHON_FOUND[@]}" --version 2>&1))" + # line-cdp-capture.py records request bodies and SSE events through CDP. + if "${PYTHON_FOUND[@]}" -c "import websocket" >/dev/null 2>&1; then + ok "Python package websocket-client" + else + need "Python package websocket-client (pip name: websocket-client), for ${PYTHON_FOUND[*]}" + fi +else + need "Python 3.8+" +fi + +exit $missing diff --git a/scripts/line-cdp-capture.py b/.agents/skills/line-capture/scripts/line-cdp-capture.py similarity index 74% rename from scripts/line-cdp-capture.py rename to .agents/skills/line-capture/scripts/line-cdp-capture.py index 142befe9..c76b41e0 100755 --- a/scripts/line-cdp-capture.py +++ b/.agents/skills/line-capture/scripts/line-cdp-capture.py @@ -4,15 +4,18 @@ Connects to Chrome's remote debugging port and captures: - SSE EventSource messages (the critical piece net-log misses) - Thrift RPC request/response bodies -- All LINE-domain request headers +- All LINE-domain request headers, including the ones Chrome adds itself + (Network.requestWillBeSentExtraInfo) +- Request bodies of every LINE request, binary ones (image uploads) included + (stored as size + sha256, and the bytes in base64 in this raw capture) Usage: # First, launch Chrome with --remote-debugging-port=9222 # Then: - python3 scripts/line-cdp-capture.py [output.json] + python3 .agents/skills/line-capture/scripts/line-cdp-capture.py [output.json] # Or use the wrapper: - bash scripts/line-chrome-capture.sh [output_dir] + bash .agents/skills/line-capture/scripts/line-chrome-capture.sh [output_dir] """ import json @@ -21,6 +24,7 @@ import signal import time import base64 +import hashlib import threading from collections import defaultdict from datetime import datetime, timezone @@ -69,7 +73,10 @@ def __init__(self, output_path: str): self.thrift_responses = {} # requestId -> response body self.request_bodies = {} # requestId -> request body self.pending_body_requests = {} # msg_id -> (requestId, session_id) + self.pending_post_requests = {} # msg_id -> requestId + self.extra_headers = {} # requestId -> headers from requestWillBeSentExtraInfo self.sessions = {} # sessionId -> targetInfo + self.target_sessions = {} # targetId -> sessionId self.running = True @@ -143,6 +150,11 @@ def send_to_session(self, session_id: str, method: str, params: dict = None) -> self.ws.send(json.dumps(msg)) return mid + def release_target(self, session_id: str, target_type: str): + """Let a page-like target run if it is held waiting for a debugger.""" + if target_type in ("page", "iframe", "other"): + self.send_to_session(session_id, "Runtime.runIfWaitingForDebugger") + def handle_event(self, msg: dict): method = msg.get("method", "") params = msg.get("params", {}) @@ -162,18 +174,39 @@ def handle_event(self, msg: dict): "maxTotalBufferSize": 100 * 1024 * 1024, "maxResourceBufferSize": 10 * 1024 * 1024, }) + # Windows opened by the extension (the per-group popup.html, the + # Settings window) can arrive held at "waiting for debugger" even + # with waitForDebuggerOnStart off, and Chrome does not always set + # waitingForDebugger on the event. Left alone they stay blank and + # grey out the extension with "Debugger paused in another tab". + # runIfWaitingForDebugger is a no-op on a running target, so send + # it to every page-like target whatever the flag says. + self.release_target(sid, ttype) # Track session -> target mapping self.sessions[sid] = target_info + self.target_sessions[target_info.get("targetId", "")] = sid + return + + # A window can start waiting after attach (it attaches with an empty + # URL and navigates), so release again whenever its info changes. + if method == "Target.targetInfoChanged": + info = params.get("targetInfo", {}) + sid = self.target_sessions.get(info.get("targetId", "")) + if sid: + self.release_target(sid, info.get("type", "")) return if method == "Target.detachedFromTarget": sid = params.get("sessionId", "") target = self.sessions.pop(sid, {}) + self.target_sessions.pop(target.get("targetId", ""), None) print(f" Detached from [{target.get('type', '?')}] {target.get('title', '?')}", file=sys.stderr) return if method == "Network.requestWillBeSent": self._on_request(params, session_id) + elif method == "Network.requestWillBeSentExtraInfo": + self._on_request_extra_info(params) elif method == "Network.responseReceived": self._on_response(params, session_id) elif method == "Network.loadingFinished": @@ -186,6 +219,27 @@ def handle_event(self, msg: dict): def handle_response(self, msg: dict): mid = msg.get("id") + if mid in self.pending_post_requests: + request_id = self.pending_post_requests.pop(mid) + result = msg.get("result", {}) + post_data = result.get("postData") + if post_data is not None: + # Newer Chrome flags binary bodies with base64Encoded; older + # ones return text only, which is lossy for binary data. + if result.get("base64Encoded"): + raw = base64.b64decode(post_data) + else: + raw = post_data.encode("utf-8", errors="surrogateescape") + self.request_bodies[request_id] = { + "size": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + "base64": base64.b64encode(raw).decode("ascii"), + "text": raw.decode("utf-8") if self._is_text(raw) else None, + } + else: + error = msg.get("error", {}).get("message", "unavailable") + self.request_bodies[request_id] = {"error": error} + return if mid in self.pending_body_requests: request_id, session_id = self.pending_body_requests.pop(mid) result = msg.get("result", {}) @@ -216,16 +270,33 @@ def _on_request(self, params: dict, session_id: str = ""): "session_id": session_id, } - # Capture POST body for Thrift calls - if request.get("hasPostData") and "/thrift/" in url: - post_data = request.get("postData", "") - if post_data: - self.request_bodies[request_id] = post_data + # Request bodies: the inline postData is missing for large or binary + # bodies (e.g. obs/uploadProfile), so always ask for it explicitly. + if request.get("hasPostData"): + target = session_id + if target: + mid = self.send_to_session(target, "Network.getRequestPostData", {"requestId": request_id}) + else: + mid = self.send("Network.getRequestPostData", {"requestId": request_id}) + self.pending_post_requests[mid] = request_id count = len(self.requests) if count % 10 == 1: print(f" [{count} LINE requests captured]", file=sys.stderr) + @staticmethod + def _is_text(raw: bytes) -> bool: + try: + raw.decode("utf-8") + except UnicodeDecodeError: + return False + return b"\x00" not in raw + + def _on_request_extra_info(self, params: dict): + """Headers as sent on the wire (Cookie, Origin, Sec-*, X-Line-* ...).""" + request_id = params.get("requestId") + self.extra_headers.setdefault(request_id, {}).update(params.get("headers", {})) + def _on_response(self, params: dict, session_id: str = ""): request_id = params.get("requestId") if request_id not in self.requests: @@ -295,6 +366,8 @@ def save(self): entry.pop("session_id", None) # internal, don't export if rid in self.thrift_responses: entry["response_body"] = self.thrift_responses[rid] + if rid in self.extra_headers: + entry["request_extra_headers"] = self.extra_headers[rid] if rid in self.request_bodies: entry["request_body"] = self.request_bodies[rid] requests_list.append(entry) diff --git a/scripts/line-chrome-capture.sh b/.agents/skills/line-capture/scripts/line-chrome-capture.sh similarity index 70% rename from scripts/line-chrome-capture.sh rename to .agents/skills/line-capture/scripts/line-chrome-capture.sh index 101967ed..627dac8e 100755 --- a/scripts/line-chrome-capture.sh +++ b/.agents/skills/line-capture/scripts/line-chrome-capture.sh @@ -2,7 +2,9 @@ # Launch Chrome with CDP-based network capture for LINE extension analysis. # # Usage: -# ./scripts/line-chrome-capture.sh [output_dir] +# bash .agents/skills/line-capture/scripts/line-chrome-capture.sh [output_dir] +# +# The script reads the bridge's ExtensionVersion from pkg/line/client.go. # # This starts Chrome with remote debugging and runs a CDP listener that # captures all LINE traffic including SSE EventSource message bodies. @@ -29,13 +31,63 @@ # This is useful when installing LINE from the Chrome Web Store into the # persistent capture profile. # -# When done, close Chrome (Cmd+Q). The capture files will be at: +# The browser opens on about:blank; open LINE from the extension's toolbar icon +# (LINE_CAPTURE_START_URL overrides the start URL). When done, quit Chrome +# completely. The capture files will be at: # /line-cdp-capture.json — CDP capture (SSE + request bodies) # /line-net-log.json — Chrome net-log (low-level backup) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# .agents/skills/line-capture/scripts -> repository root +REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)" + +# Without its dependencies the CDP listener dies silently and only the net-log +# is saved, which lacks request bodies and SSE events. +if ! "$SCRIPT_DIR/line-capture-preflight.sh" >/dev/null; then + "$SCRIPT_DIR/line-capture-preflight.sh" + echo "" + echo "==> Install the missing dependencies above, then run this again." + exit 1 +fi +# Sets CHROME and the PYTHON array (it can be several words, like "py -3"). +eval "$("$SCRIPT_DIR/line-capture-preflight.sh" --print-env)" + +# Windows Chrome cannot use WSL paths or reach the recorder inside WSL. +if grep -qi microsoft /proc/version 2>/dev/null && [[ "$CHROME" == *.exe ]]; then + echo "Windows chrome.exe does not work from WSL. Use a Linux Chrome or Chromium in WSL, or run this from Git Bash on Windows." >&2 + exit 1 +fi + +warn() { echo " WARNING: $*" >&2; } + +# Chrome on Windows needs native paths; Git Bash paths only convert implicitly +# for some arguments. +native_path() { + if command -v cygpath >/dev/null 2>&1; then + cygpath -w "$1" + else + echo "$1" + fi +} + +# Directories that can hold the user's normal Chrome profiles, per OS. +normal_chrome_dirs() { + case "$(uname -s)" in + Darwin) + echo "$HOME/Library/Application Support/Google/Chrome" ;; + MINGW* | MSYS* | CYGWIN*) + local local_appdata="${LOCALAPPDATA:-$HOME/AppData/Local}" + command -v cygpath >/dev/null 2>&1 && local_appdata="$(cygpath -u "$local_appdata")" + echo "$local_appdata/Google/Chrome/User Data" ;; + *) + echo "$HOME/.config/google-chrome" + echo "$HOME/.config/chromium" ;; + esac +} + OUTPUT_DIR="${1:-/tmp/line-chrome-extension-capture}" mkdir -p "$OUTPUT_DIR" NETLOG_FILE="$OUTPUT_DIR/line-net-log.json" @@ -56,7 +108,7 @@ seed_chrome_capture_profile() { # below keep the choices explicit and readable for future maintainers. touch "$profile_dir/First Run" - python3 - "$profile_dir" <<'PY' + "${PYTHON[@]}" - "$profile_dir" <<'PY' import json import sys from pathlib import Path @@ -139,10 +191,20 @@ PY find_latest_extension_dir() { local extension_id="$1" - local base="$HOME/Library/Application Support/Google/Chrome/Default/Extensions/$extension_id" - if [ -d "$base" ]; then - find "$base" -mindepth 1 -maxdepth 1 -type d | sort -V | tail -n 1 - fi + local chrome_dir base found + while IFS= read -r chrome_dir; do + base="$chrome_dir/Default/Extensions/$extension_id" + [ -d "$base" ] || continue + # Chrome's folders can be unreadable (macOS privacy protection answers + # "Operation not permitted"); skip those instead of aborting. + found="$(find "$base" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -n 1)" + if [ -n "$found" ]; then + # First usable match only: Chrome and Chromium can both have it. + echo "$found" + return 0 + fi + warn "cannot read $base, skipping" + done < <(normal_chrome_dirs) } find_latest_profile_extension_dir() { @@ -150,14 +212,14 @@ find_latest_profile_extension_dir() { local extension_id="$2" local base="$profile_dir/Default/Extensions/$extension_id" if [ -d "$base" ]; then - find "$base" -mindepth 1 -maxdepth 1 -type d | sort -V | tail -n 1 + find "$base" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -V | tail -n 1 || true fi } read_manifest_version() { local extension_dir="$1" if [ -n "${extension_dir:-}" ] && [ -f "$extension_dir/manifest.json" ]; then - python3 - "$extension_dir/manifest.json" <<'PY' + "${PYTHON[@]}" - "$extension_dir/manifest.json" <<'PY' import json import sys from pathlib import Path @@ -171,9 +233,9 @@ PY } find_repo_extension_version() { - local client_file="$SCRIPT_DIR/../pkg/line/client.go" + local client_file="$REPO_ROOT/pkg/line/client.go" if [ -f "$client_file" ]; then - python3 - "$client_file" <<'PY' + "${PYTHON[@]}" - "$client_file" <<'PY' import re import sys from pathlib import Path @@ -193,7 +255,7 @@ stage_extension_dir() { fi local version - version="$(python3 - "$source_dir/manifest.json" <<'PY' + version="$("${PYTHON[@]}" - "$source_dir/manifest.json" <<'PY' import json import sys from pathlib import Path @@ -211,13 +273,21 @@ PY # Chrome Web Store extension directories contain _metadata/, which Chrome # rejects when loading as unpacked. Stage a clean copy for capture runs. + # An unreadable source must not abort the capture; skip with a warning. if command -v rsync >/dev/null 2>&1; then - rsync -a --delete --exclude '_metadata' "$source_dir/" "$dest/" + rsync -a --delete --exclude '_metadata' "$source_dir/" "$dest/" 2>/dev/null || { + warn "could not copy $source_dir, skipping this extension" + return 0 + } else local tmp="$dest.tmp" rm -rf "$tmp" mkdir -p "$tmp" - (cd "$source_dir" && tar --exclude './_metadata' -cf - .) | (cd "$tmp" && tar -xf -) + if ! { (cd "$source_dir" && tar --exclude './_metadata' -cf - .) | (cd "$tmp" && tar -xf -); } 2>/dev/null; then + rm -rf "$tmp" + warn "could not copy $source_dir, skipping this extension" + return 0 + fi rm -rf "$dest" mv "$tmp" "$dest" fi @@ -250,13 +320,6 @@ if [ -n "${CODEX_EXTENSION_DIR:-}" ] && [ -f "$CODEX_EXTENSION_DIR/manifest.json EXTENSION_DIRS+=("$CODEX_EXTENSION_DIR") fi -CHROME="${CHROME:-/Applications/Google Chrome.app/Contents/MacOS/Google Chrome}" -if [ ! -x "$CHROME" ]; then - echo "Chrome not found at $CHROME" - echo "Set CHROME env var to your Chrome binary path." - exit 1 -fi - FIRST_RUN=false if [ ! -d "$DEBUG_PROFILE" ]; then FIRST_RUN=true @@ -298,8 +361,9 @@ if [ "$FIRST_RUN" = true ]; then echo " The profile persists across capture dirs — you only need to do this once." echo "" fi -echo " 1. Open the LINE Chrome Extension and do your flow" -echo " 2. When done, close Chrome (Cmd+Q)" +echo " 1. Click the LINE extension icon in the toolbar (puzzle-piece menu if not pinned) and do your flow" +echo " Do not open DevTools on the extension: LINE has debugger traps and the recorder does not need it." +echo " 2. When done, quit Chrome completely (all windows, not just the tab)" echo "" # Chrome 146+ requires --user-data-dir for remote debugging. @@ -310,14 +374,18 @@ CHROME_ARGS=( --disable-features=SignInPromo,SigninInterception,ChromeSigninIntercept,DiceWebSigninInterception --remote-debugging-port=9222 --remote-allow-origins="*" - --user-data-dir="$DEBUG_PROFILE" - --log-net-log="$NETLOG_FILE" + --user-data-dir="$(native_path "$DEBUG_PROFILE")" + --log-net-log="$(native_path "$NETLOG_FILE")" --net-log-capture-mode=IncludeSensitive ) if [ ${#EXTENSION_DIRS[@]} -gt 0 ]; then + NATIVE_EXTENSION_DIRS=() + for ext_dir in "${EXTENSION_DIRS[@]}"; do + NATIVE_EXTENSION_DIRS+=("$(native_path "$ext_dir")") + done IFS=, - EXTENSION_ARG="${EXTENSION_DIRS[*]}" + EXTENSION_ARG="${NATIVE_EXTENSION_DIRS[*]}" unset IFS if [ -z "${LINE_PROFILE_EXTENSION_DIR:-}" ]; then CHROME_ARGS+=(--disable-extensions-except="$EXTENSION_ARG") @@ -325,22 +393,15 @@ if [ ${#EXTENSION_DIRS[@]} -gt 0 ]; then CHROME_ARGS+=(--load-extension="$EXTENSION_ARG") fi -START_URL="${LINE_CAPTURE_START_URL:-}" -if [ -z "$START_URL" ] && [ -n "${LINE_PROFILE_EXTENSION_DIR:-}" ] && [ -f "$LINE_PROFILE_EXTENSION_DIR/manifest.json" ]; then - START_URL="chrome-extension://ophjlpahpchlmihnnnihgmmeilfjmjjc/index.html" -fi -if [ -z "$START_URL" ] && [ -n "${LINE_EXTENSION_DIR:-}" ] && [ -f "$LINE_EXTENSION_DIR/manifest.json" ]; then - START_URL="chrome-extension://ophjlpahpchlmihnnnihgmmeilfjmjjc/index.html" -fi -if [ -z "$START_URL" ]; then - START_URL="about:blank" -fi +# about:blank on purpose: opening the extension's index.html as a tab behaves +# differently from the real popup, so the user opens LINE from the toolbar icon. +START_URL="${LINE_CAPTURE_START_URL:-about:blank}" "$CHROME" "${CHROME_ARGS[@]}" "$START_URL" 2>/dev/null & CHROME_PID=$! # Start the CDP capture (it waits for Chrome to be ready) -python3 "$SCRIPT_DIR/line-cdp-capture.py" "$CDP_FILE" & +"${PYTHON[@]}" "$SCRIPT_DIR/line-cdp-capture.py" "$CDP_FILE" & CDP_PID=$! # Wait for Chrome to exit diff --git a/.agents/skills/line-implement/REFERENCE.md b/.agents/skills/line-implement/REFERENCE.md index 649075ea..e6827ec3 100644 --- a/.agents/skills/line-implement/REFERENCE.md +++ b/.agents/skills/line-implement/REFERENCE.md @@ -5,8 +5,8 @@ Start with: - `AGENTS.md` / `CLAUDE.md` -- `.agents/commands/line-capture.md` -- `.agents/commands/line-analyze.md` +- `.agents/skills/line-capture/SKILL.md` +- `.agents/skills/line-analyze/SKILL.md` - `.agents/commands/line-implement.md` - `docs/protocol/README.md` - `docs/protocol/endpoints/README.md` @@ -61,7 +61,7 @@ IDs, MIDs, contacts, message bodies, media object IDs, QR/login artifacts, and E2EE metadata. - Use `/tmp/line-chrome-extension-capture/-` with `umask 077`. -- Prefer the persistent dedicated Chrome profile created by `scripts/line-chrome-capture.sh` +- Prefer the persistent dedicated Chrome profile created by `.agents/skills/line-capture/scripts/line-chrome-capture.sh` at `~/.cache/line-chrome-extension-capture/chrome-debug-profile`. - Ask before using the user's normal Chrome profile. - Ask before stopping a running bridge or invalidating an active LINE session. @@ -88,7 +88,7 @@ Use CDP capture first because it includes SSE event bodies: ```bash umask 077 CAPTURE_DIR="/tmp/line-chrome-extension-capture/$(date +%Y%m%d-%H%M%S)-feature-slug" -bash scripts/line-chrome-capture.sh "$CAPTURE_DIR" +bash .agents/skills/line-capture/scripts/line-chrome-capture.sh "$CAPTURE_DIR" ``` The launcher stages clean unpacked copies of the LINE and Codex Chrome extension @@ -112,15 +112,15 @@ sign-in screen, choose "Stay Signed Out" and continue to LINE login only. When Chrome closes: ```bash -python3 scripts/parse-line-traffic.py "$CAPTURE_DIR/line-cdp-capture.json" "$CAPTURE_DIR/line-cdp-capture-parsed.json" -python3 scripts/redact-line-capture.py "$CAPTURE_DIR/line-cdp-capture-parsed.json" +python3 .agents/skills/line-analyze/scripts/parse-line-traffic.py "$CAPTURE_DIR/line-cdp-capture.json" "$CAPTURE_DIR/line-cdp-capture-parsed.json" +python3 .agents/skills/line-analyze/scripts/redact-line-capture.py "$CAPTURE_DIR/line-cdp-capture-parsed.json" ``` Use the net-log only as a backup: ```bash -python3 scripts/parse-line-traffic.py "$CAPTURE_DIR/line-net-log.json" "$CAPTURE_DIR/line-net-log-parsed.json" -python3 scripts/redact-line-capture.py "$CAPTURE_DIR/line-net-log-parsed.json" +python3 .agents/skills/line-analyze/scripts/parse-line-traffic.py "$CAPTURE_DIR/line-net-log.json" "$CAPTURE_DIR/line-net-log-parsed.json" +python3 .agents/skills/line-analyze/scripts/redact-line-capture.py "$CAPTURE_DIR/line-net-log-parsed.json" ``` ## Analysis Checklist diff --git a/.claude/skills b/.claude/skills new file mode 120000 index 00000000..2b7a412b --- /dev/null +++ b/.claude/skills @@ -0,0 +1 @@ +../.agents/skills \ No newline at end of file From f60805659474ce386dc58f56ac3b9b20b72c1d75 Mon Sep 17 00:00:00 2001 From: Caio Lins Date: Tue, 27 Oct 2026 23:57:21 +0900 Subject: [PATCH 2/3] tooling: add the line-test-plan skill Writes the numbered test plan (test//PLAN.md) for a bridge feature before any capture or code: lettered sections, stable step IDs, before/capture/after phases and a Result column, plus labeled sample media when the feature needs files. check-plan.py validates the layout and make-samples.py generates the samples. Co-Authored-By: Claude Opus 5.5 --- .agents/skills/line-test-plan/SKILL.md | 145 ++++++++ .agents/skills/line-test-plan/TEMPLATE.md | 117 +++++++ .../line-test-plan/scripts/check-plan.py | 114 +++++++ .../scripts/line-test-plan-preflight.sh | 93 +++++ .../line-test-plan/scripts/make-samples.py | 323 ++++++++++++++++++ 5 files changed, 792 insertions(+) create mode 100644 .agents/skills/line-test-plan/SKILL.md create mode 100644 .agents/skills/line-test-plan/TEMPLATE.md create mode 100644 .agents/skills/line-test-plan/scripts/check-plan.py create mode 100644 .agents/skills/line-test-plan/scripts/line-test-plan-preflight.sh create mode 100644 .agents/skills/line-test-plan/scripts/make-samples.py diff --git a/.agents/skills/line-test-plan/SKILL.md b/.agents/skills/line-test-plan/SKILL.md new file mode 100644 index 00000000..d8614c4e --- /dev/null +++ b/.agents/skills/line-test-plan/SKILL.md @@ -0,0 +1,145 @@ +--- +name: line-test-plan +description: Writes the numbered test plan (PLAN.md) for a beeper/line bridge feature before any capture or code, with lettered sections, step IDs, phases (before, capture, after) and Result columns, plus labeled sample media when the feature needs files. Use when starting a new LINE bridge feature, when asked for a test plan or capture plan, or before running line-capture. +--- + +# LINE Test Plan + +Writes `test//PLAN.md` inside a `beeper/line` checkout: the list of +things to do in LINE, Beeper and the Chrome Extension, numbered so every +screenshot, request and result can point at a step. It comes before capture and +code. Run the commands below from the repository root. + +## Contract + +Other skills read this layout (`line-evidence` relies on it), so keep it exact. + +- Feature folder: `test//` in the checkout. `/test` is gitignored there; nothing in it is committed. +- `PLAN.md` holds, in this order: + 1. title, goal, and the issue/PR link + 2. **Scope**: copy the LINE Chrome Extension, nothing extra + 3. **Platforms** involved (Chrome extension, Beeper Desktop, Beeper mobile, LINE iOS, LINE Android) and **Test chats** (which chats and accounts exist) + 4. **Coverage** and **Not testable** (known gaps): see "Coverage and gaps" below + 5. `## Test data`: a bullet per dummy string used in tests (group names, message text, sample file labels). `line-evidence` treats this as the allowlist of text that is not PII + 6. `## Run order`: which sections run in which phase + 7. lettered sections `## A. Title`, each with one table + 8. `## Findings`: a link to `FINDINGS.md` (the agent writes it in the analyze phase, after `line-analyze` has parsed the capture; the plan only links to it) +- Step IDs match `^[A-Z][0-9]+$` (`A1`, `B12`) and are unique across the plan. +- Every table has exactly these columns: `#`, `Platform`, `Phase`, `Action`, `Checks`, `Evidence`, `Result`. + - `Phase` is `before`, `capture` or `after`. + - `Evidence` names the screenshots and requests the step should produce, for example `shot: Beeper shows the GIF; req: sendMessage`. + - `Result` starts empty. +- Step IDs are stable. Once any `Result` cell is filled, never re-letter or renumber sections or steps: results, screenshots, `FINDINGS.md` and `line-evidence` all point at the IDs. Add new sections with the next free letter, and new steps with the next free number in their section (`A3b` is not valid: use `A9`). Mark a dropped step `n/a` with the reason instead of deleting it. Re-lettering is fine only while no results exist. +- Sample media lives in `test//samples/`. + +[TEMPLATE.md](TEMPLATE.md) is a skeleton with every part. + +## What to do + +1. **Ask for the goal and the issue or PR.** One question: the feature in a + sentence, and the `beeper/line` issue or PR number (read it with `gh issue + view` / `gh pr view`). Pick a short `` (`group-avatar`, + `send-gifs`) and confirm it. + +2. **Ask about the test setup.** The shape of the plan depends on this, and + guessing it cost the first real run (GIF sending) three rewrites. Ask in one + message, and write the answers into `## Platforms` and `## Test chats`: + - **Chats and accounts:** which test chats exist: a group, a 1:1 chat, a chat with a business account or bot. Whether the chats are end-to-end encrypted (Letter Sealing) decides the upload path, so say which is which. + - **Devices:** which are available: Chrome extension, Beeper Desktop, Beeper iOS/Android, LINE iOS, LINE Android. + - **Second account:** is there a second real LINE account, ideally on its own phone, for the receiver's view? If not, that is a known gap (see "Sender vs. receiver"). + - **What the user already knows:** for example which Beeper clients can already do the thing, known limits, earlier findings. Write these into `## Scope` as "Known before testing" so no step re-tests them. + +3. **Read what exists, then propose.** + - `AGENTS.md`/`CLAUDE.md`, `docs/protocol/`, and the code the feature touches in `pkg/line/` and `pkg/connector/`. + - Every `test/*/FINDINGS.md` of related features: what the extension already + enforces (limits, formats, rules) tells you which cases are known and which + still need a capture. + - Then show the user the proposed sections as a short list (letter, title, + phase, platform, number of steps) and let them add, drop or reorder before + you write steps. Do not write the full plan first. + +4. **Sections to propose.** Always include: + - **Beeper baseline** (`before`): how the feature behaves today through the bridge, so "after" has something to compare with. + - **LINE's own clients as the reference** (`before`): do the same thing from LINE iOS or Android (the same sample file, the same chat) and look at the result in every client. This needs no capture and is often the best reference for what "correct" looks like. Not everything on the LINE side needs the Chrome capture; use the capture only for what the Chrome extension sends or receives on the wire. + - **Sending from Chrome** (`capture`): one step per variant the extension offers. + - **Changes made on another device** (`capture`): the user does it in LINE on their phone while the extension stays open; this is what the bridge receives. Include LINE iOS and Android if both are available. Add a step for "undo it" so the test chat is tidy. + - **Edge cases** (`capture`): limits (length, size, count), empty or unchanged input, cancel before sending, unsupported formats (greyed out in a picker is a result too), unicode and emoji, group vs. 1:1 chat, Letter Sealing on vs. off. + - **Beeper after deploy** (`after`): the same actions as the baseline, plus the edge cases that matter, plus changes made on LINE iOS/Android arriving in Beeper (Desktop and mobile). + - **LINE app settings on the receiving side** (for media features: images, GIFs, video, files). See "LINE app settings" below. + - Add sections the feature needs (media formats, reactions, permissions). Cover only what the extension does. + - Prefer one phase per section. If a topic spans phases, split it into two sections and cross-reference them. + +5. **Order the phases around the single Chrome Extension session.** LINE allows + one extension session at a time, so a capture logs the bridge out (and + running the bridge logs the capture out). + 1. `before`: Beeper steps, bridge logged in. + 2. `capture`: the user logs the bridge out, starts `line-capture`, and does every Chrome step and every other-device step without breaking the session; then quits Chrome. + 3. `after`: the feature is deployed, the bridge is logged in again, and the Beeper steps run on the new build. + Write this into `## Run order` (phase → sections). Within a table, rows follow the order the user performs them. + +6. **Write the steps.** Each one is a single action a person can do in about 10 + seconds, with a `Checks` cell that says what the step is for ("which + attribute flag", "is the 60-character name cut?") and an `Evidence` cell with + the expected screenshot or request. Use only dummy strings, and list every + one in `## Test data`. Never put real names, MIDs, phone numbers or tokens + in the plan. + +7. **Sample media, when the feature needs files.** + - Run `bash .agents/skills/line-test-plan/scripts/line-test-plan-preflight.sh` (on Windows, from Git Bash). It prints `OK`, `MISSING` (required) or `OPTIONAL` per dependency and does not say how to install anything: choose the install method that fits the detected OS and package managers (for example ` -m pip install --user Pillow`), show the commands, ask the user to confirm, install, and re-run until it exits 0. + - Use the Python the preflight found (`py -3` on Windows only; a `PYTHON` override is respected): `eval "$(bash .agents/skills/line-test-plan/scripts/line-test-plan-preflight.sh --print-env)"` sets the `PYTHON` array, and the commands below run it as `"${PYTHON[@]}"`. + - Generate labeled samples: + ```bash + "${PYTHON[@]}" .agents/skills/line-test-plan/scripts/make-samples.py test//samples --kinds image,animated + ``` + `--kinds` takes `image`, `animated`, `video`, `shape`, `ladder`; `--only 10,21` picks files by number. The animated set includes a transparent GIF, a tiny 32×32 one, and wide, tall and square ones. `ladder` is the size ladder (about 5, 10, 20 and 27 MB of animated GIF, about 60 MB in all, so it is not built by default): use `--kinds ladder` or `--only 28,29,30,24`, and send the ladder smallest first. Every file shows its own name and corner markers, so a screenshot proves which file was sent and how it was cropped. (Samples under 64 px show only their number, such as `23`, because the name does not fit; `MANIFEST.txt` maps it to the file.) + - It runs on macOS, Linux and Windows with Python 3.8+ and Pillow. Video (MP4, HEVC, MOV, WebM) needs `ffmpeg`; HEIC and AVIF need `pillow-heif` or an `ffmpeg` with the encoder. Whatever it cannot encode is listed as skipped: tell the user, and do not put those files in the plan. + - Put each file name in the plan's steps and its label (from `samples/MANIFEST.txt`) in `## Test data`. + +8. **Validate and hand over.** Run `"${PYTHON[@]}" .agents/skills/line-test-plan/scripts/check-plan.py test//PLAN.md` (set `PYTHON` as in step 7; do it here too if the feature has no media) and fix what it reports. Then tell the user how to run it: + - **Pacing:** do one step, wait about 10 seconds, then the next. The gaps let the analysis match each step to its network calls. Other-device steps need about 15 seconds before the next one. + - **Result column:** say it as you go ("A3 saved", "B4 greyed out", "C2 error: ..."). Mark `✅` accepted or working, `❌` rejected or broken (copy the error text), `n/a` with the reason when the step cannot be done, and add notes on cropping, animation or timing. + - **Order:** all `before` steps, then log the bridge out and start the capture for the `capture` steps, then deploy and run the `after` steps. + +9. **Fill results as the user reports them.** When the user says what happened, + edit the matching `Result` cell right away, with the step ID. If a report + does not match any step, ask. Keep errors verbatim, and leave a step empty + rather than guessing. When the plan is finished, point to `line-capture` for + the capture phase and `line-analyze` to parse it; the agent then writes `FINDINGS.md` from that analysis. + +## Coverage and gaps + +Put both in the plan, after `## Test chats` and before `## Test data`. + +- **Coverage table:** who sends (rows) by where you can see it (columns). For example senders Beeper Desktop, Beeper mobile, LINE iOS, Chrome extension, bot relay; viewers LINE iOS, LINE Android, Chrome extension, Beeper Desktop, Beeper mobile. Each cell is a step ID range, `-` for not applicable, or `gap` for not testable. A reader sees at a glance what is covered. +- **Known gaps / not testable:** a bullet per thing this run cannot check, with the reason (no second account, no Android device, no way to send it) and what would close it. Gaps are findings too; carry them into `FINDINGS.md`. + +## Sender vs. receiver + +LINE can show your own sends differently from what others get. For example LINE iOS only autoplayed GIFs sent from that phone, while a GIF sent from another device of the same account (the Chrome extension or the bridge) needed a tap, so the sender's own view proves little. + +- A real second LINE account on its own phone is the true receiver test. Plan the receiver steps for it, and list them under gaps when there is none. +- A Messaging API relay bot is **not** a valid receiver for media. It re-sends the file with its own metadata and can flatten things (GIFs lose animation attributes), so what you see is the bot's message, not the bridge's. Use it only to get a received-message view, and say so in the plan. +- The Chrome extension opened on the chat is a received-message view with no local file cache; it is a useful second opinion, not a replacement for a second account. + +## LINE app settings + +For media features add a section of the receiving side's LINE app settings (autoplay, auto-download, photo quality, and so on). If `readme/LINE_CLIENT_SETTINGS.md` exists in the checkout, read it and use it as the list of settings to consider; if not, take the list from the app's Settings screens. Do not invent settings. + +- Pick only those that can change what the feature shows. +- Before the run, write down each setting's current value (a `Settings record` line under `## Run order`), and end the section with a step that puts them back. +- For each setting value, send one copy from LINE and one from Beeper (or the bridge) and compare how each arrives. Change one setting at a time. + +## Sample files on the phone + +When a mobile app is involved (Beeper or LINE on iOS or Android), add a step at the start of the first section that uses it: put `test//samples/` on the phone (AirDrop to the iPhone and save to Files; on Android, a cable, Files, or a cloud folder), so both apps attach the same files. Without it the phone sends other files than the ones named in the plan. + +## Recording bridge logs + +Raise the bridge's log level (to debug) for the run, and note the current value in the plan so it can be restored. Start the bridge's log output into a file under `test//` before the `before` and `after` phases, and stop and restore the level when the phase ends. Say in the plan how the logs are read (where they go, which time window per step). Never paste logs into the plan or a PR without redaction. + +## Visual overview (optional) + +Offer a one-page visual overview of the plan: the phases, sections, coverage table and gaps, built from `PLAN.md` as a single self-contained HTML file. Ask first; many people don't need one. Use only the plan's dummy test data, no real names, MIDs, numbers or screenshots. Keep it local in the feature folder, and publish it (for example as an Artifact) only when the user says so. + +Never commit the feature folder, and keep captures, tokens, MIDs and real +account data out of the plan. diff --git a/.agents/skills/line-test-plan/TEMPLATE.md b/.agents/skills/line-test-plan/TEMPLATE.md new file mode 100644 index 00000000..1de84cf8 --- /dev/null +++ b/.agents/skills/line-test-plan/TEMPLATE.md @@ -0,0 +1,117 @@ +# Test plan: (beeper/line#) + +**Goal:** + +## Scope + +Copy what the LINE Chrome Extension does, nothing extra. Anything the extension cannot do is out of scope; write down that it cannot. + +Known before testing: +- + +## Platforms + +- Chrome extension (LINE Chrome Extension ) +- Beeper Desktop +- Beeper mobile +- LINE iOS +- LINE Android + +Drop the ones you don't have; list devices and OS versions only as test data allows. + +## Test chats + +- ``: +- `<1:1 chat>`: <...> +- ``: +- Second account for the receiver's view: + +Use test chats. Every change may post a notice that all members see. + +**Before A** (when a phone app is involved): put `test//samples/` on the phone (AirDrop and save to Files, or Files on Android), so every app attaches the same files. + +## Coverage + +Who sends (rows) by where you can see it (columns). Cells are step ID ranges; `-` is not applicable; `gap` is listed under Not testable. + +| Sender \ Viewer | LINE iOS | LINE Android | Chrome extension | Beeper Desktop | +|---|---|---|---|---| +| Beeper Desktop (before) | A1 | gap | - | A1 | +| LINE iOS (reference) | E1 | gap | C1 | - | +| Beeper Desktop (settings) | E2 | gap | - | - | +| Chrome extension | D1 | gap | B1 | - | +| Beeper Desktop (after) | F1 | gap | - | F1 | + +Keep the rows and columns to what the plan has, and make every cell point at a real step. + +## Not testable + +Known gaps in this run, each with the reason and what would close it: +- + +## Test data + +Every dummy string used in the tests. `line-evidence` treats these as safe text. + +- `` +- `` +- `` + +## Run order + +LINE allows one Chrome Extension session, so the capture logs the bridge out. + +1. `before`: Beeper, bridge logged in: +2. `capture`: log the bridge out, start `line-capture`, then: . Quit Chrome when done. +3. `after`: deploy, log the bridge in again: + +**Settings record:** (media features) write down the LINE app settings before the first step, and put them back in the last step of the settings section. Bridge log level: ; raise it for the run, restore it after. + +**Step IDs are stable:** once a Result is filled, never re-letter or renumber. Add sections and steps with the next free letter or number. + +**Pacing:** wait about 10 s between steps (about 15 s after a step on another device). Write what happened in the **Result** column: ✅ worked, ❌ rejected (copy the error text), `n/a` with the reason, plus notes. + +## A. Beeper baseline today + +| # | Platform | Phase | Action | Checks | Evidence | Result | +|---|----------|-------|--------|--------|----------|--------| +| A1 | Beeper Desktop | before | | | shot: | | + +## B. Sending from Chrome + +| # | Platform | Phase | Action | Checks | Evidence | Result | +|---|----------|-------|--------|--------|----------|--------| +| B1 | Chrome extension | capture | | | shot: ; req: | | + +## C. Changes made on another device + +Keep the extension open on the test chat. Make each change in the LINE app on your phone, then wait about 15 s. This is what the bridge receives. + +| # | Platform | Phase | Action | Checks | Evidence | Result | +|---|----------|-------|--------|--------|----------|--------| +| C1 | LINE iOS | capture | | | shot: Chrome shows it; req: | | + +## D. Edge cases + +| # | Platform | Phase | Action | Checks | Evidence | Result | +|---|----------|-------|--------|--------|----------|--------| +| D1 | Chrome extension | capture | | blocked in the UI or by the server? | shot: | | + +## E. LINE app settings on the receiving side + +Media features only. Settings to consider are listed in `readme/LINE_CLIENT_SETTINGS.md` of the `beeper/line` checkout, when it exists. Change one setting at a time; send one copy from LINE and one from Beeper under each value, and compare. Last step restores the settings. + +| # | Platform | Phase | Action | Checks | Evidence | Result | +|---|----------|-------|--------|--------|----------|--------| +| E1 | LINE iOS | before | | | shot: LINE iOS | | +| E2 | LINE iOS | before | | | shot: LINE iOS | | + +## F. Beeper after deploy + +| # | Platform | Phase | Action | Checks | Evidence | Result | +|---|----------|-------|--------|--------|----------|--------| +| F1 | Beeper Desktop | after | | matches Chrome | shot: Beeper; shot: LINE shows it | | + +## Findings + +[FINDINGS.md](FINDINGS.md) (written by the agent in the analyze phase, after `line-analyze` parses the capture). diff --git a/.agents/skills/line-test-plan/scripts/check-plan.py b/.agents/skills/line-test-plan/scripts/check-plan.py new file mode 100644 index 00000000..0cb6f56b --- /dev/null +++ b/.agents/skills/line-test-plan/scripts/check-plan.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Check that a PLAN.md follows the line-test-plan contract. + +Usage: python3 check-plan.py test//PLAN.md + +Checks: a `## Test data` section with at least one bullet; lettered sections +(`## A. Title`); tables with the columns #, Platform, Phase, Action, Checks, +Evidence, Result; step IDs matching ^[A-Z][0-9]+$ and unique; Phase one of +before | capture | after; a `## Run order` section. +Exits 1 and lists every problem when the plan does not conform. +""" + +import re +import sys + +COLUMNS = ["#", "Platform", "Phase", "Action", "Checks", "Evidence", "Result"] +PHASES = ["before", "capture", "after"] +STEP = re.compile(r"^[A-Z][0-9]+$") + + +def cells(line): + # Drop one outer pipe at each end only: a trailing "||" is an empty last cell (Result). + s = line.strip() + if s.startswith("|"): + s = s[1:] + if s.endswith("|"): + s = s[:-1] + return [c.strip() for c in s.split("|")] + + +def main(path): + problems = [] + text = open(path, encoding="utf-8").read().splitlines() + + sections = [i for i, l in enumerate(text) if re.match(r"^## [A-Z]\. ", l)] + if not sections: + problems.append("no lettered sections (## A. Title)") + in_test_data, bullets = False, 0 + for l in text: + if l.startswith("## "): + in_test_data = l.strip() == "## Test data" + elif in_test_data and l.lstrip().startswith(("- ", "* ")): + bullets += 1 + if not any(l.strip() == "## Test data" for l in text): + problems.append("missing `## Test data` section") + elif not bullets: + problems.append("`## Test data` has no bullets (list every dummy string, or `- none`)") + + seen = set() + header = None # the columns of the table we are in, or None outside a table + section = None # title of the current lettered section + rows_in_section = 0 + + def close_section(): + if section and rows_in_section == 0: + problems.append(f"section {section!r} has no step rows (missing or empty table)") + + for n, l in enumerate(text, 1): + if l.startswith("## "): + close_section() + header, rows_in_section = None, 0 + section = l[3:].strip() if re.match(r"^## [A-Z]\. ", l) else None + if not l.lstrip().startswith("|"): + if l.strip(): + header = None + continue + if section is None: + continue # tables outside lettered sections (none expected) are not steps + c = cells(l) + if c and c[0] == "#": + header = c + if c != COLUMNS: + problems.append(f"line {n}: columns are {c}, expected {COLUMNS}") + continue + if set("".join(c)) <= set("-: "): + continue + if header is None: + problems.append(f"line {n}: table row without a header row ({COLUMNS})") + continue + if len(c) != len(COLUMNS): + problems.append(f"line {n}: row has {len(c)} cells, expected {len(COLUMNS)}") + continue + rows_in_section += 1 + sid, platform, phase = c[0], c[1], c[2] + if not STEP.match(sid): + problems.append(f"line {n}: step ID {sid!r} does not match ^[A-Z][0-9]+$") + elif sid in seen: + problems.append(f"line {n}: duplicate step ID {sid}") + seen.add(sid) + if phase not in PHASES: + problems.append(f"line {n}: {sid} phase {phase!r} is not one of {PHASES}") + if not platform: + problems.append(f"line {n}: {sid} has no platform") + close_section() + + if not seen: + problems.append("no step rows found") + + order = re.search(r"^## Run order\s*$", "\n".join(text), re.M) + if not order: + problems.append("missing `## Run order` section (before, then capture, then after)") + + if problems: + print(f"{path}: {len(problems)} problem(s)") + for p in problems: + print(" -", p) + sys.exit(1) + print(f"{path}: ok, {len(seen)} steps") + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit(__doc__) + main(sys.argv[1]) diff --git a/.agents/skills/line-test-plan/scripts/line-test-plan-preflight.sh b/.agents/skills/line-test-plan/scripts/line-test-plan-preflight.sh new file mode 100644 index 00000000..44a732da --- /dev/null +++ b/.agents/skills/line-test-plan/scripts/line-test-plan-preflight.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Check what make-samples.py needs to generate labeled sample media for a test plan. +# Only needed when the feature sends or receives media. Works on macOS, Linux and +# Windows (Git Bash / MSYS2). +# +# Usage: +# .agents/skills/line-test-plan/scripts/line-test-plan-preflight.sh # human/agent-readable report +# eval "$(.agents/skills/line-test-plan/scripts/line-test-plan-preflight.sh --print-env)" # sets the PYTHON array +# +# The report prints the detected OS, one OK line per dependency found, one +# MISSING line per required dependency that is not, and one OPTIONAL line per +# optional one. It deliberately does not say how to install anything: the agent +# running it picks the install method that fits the OS and the package managers +# available, and asks the user first. +# Exits 1 when a required dependency is missing. Optional ones never fail it; +# make-samples.py skips the formats it cannot encode and says which. +# +# Override detection with PYTHON=/path/to/python (one executable; spaces are fine). + +set -uo pipefail + +case "$(uname -s)" in + Darwin) OS=macos ;; + Linux) + if grep -qi microsoft /proc/version 2>/dev/null; then OS=wsl; else OS=linux; fi ;; + MINGW* | MSYS* | CYGWIN*) OS=windows ;; + *) OS=unknown ;; +esac + +# Sets the PYTHON_FOUND array: the executable first, then optional arguments +# ("py -3"). An array keeps a PYTHON override with spaces in its path in one piece. +find_python() { + PYTHON_FOUND=() + local candidate + if [ -n "${PYTHON:-}" ]; then + "$PYTHON" -c "import sys; sys.exit(sys.version_info < (3, 8))" 2>/dev/null && PYTHON_FOUND=("$PYTHON") + return + fi + # "python3" on Windows can be the Microsoft Store stub, so the version check + # also filters out commands that exist but don't run. + for candidate in "python3" "python" "py -3"; do + # shellcheck disable=SC2206 # intentional split of "py -3" + local cmd=($candidate) + if "${cmd[@]}" -c "import sys; sys.exit(sys.version_info < (3, 8))" 2>/dev/null; then + PYTHON_FOUND=("${cmd[@]}") + return + fi + done +} + +find_python + +if [ "${1:-}" = "--print-env" ]; then + printf 'PYTHON=(' + [ ${#PYTHON_FOUND[@]} -gt 0 ] && printf '%q ' "${PYTHON_FOUND[@]}" + printf ')\n' + exit 0 +fi + +missing=0 +ok() { echo "OK $1"; } +need() { echo "MISSING $1"; missing=1; } +optional() { echo "OPTIONAL $1"; } + +echo "OS $OS ($(uname -sm))" + +if [ ${#PYTHON_FOUND[@]} -gt 0 ]; then + ok "Python 3.8+: ${PYTHON_FOUND[*]} ($("${PYTHON_FOUND[@]}" --version 2>&1))" + if "${PYTHON_FOUND[@]}" -c "import PIL" >/dev/null 2>&1; then + ok "Python package Pillow (images: JPEG, PNG, BMP, TIFF, GIF, WebP, APNG)" + if "${PYTHON_FOUND[@]}" -c "import pillow_heif" >/dev/null 2>&1; then + ok "Python package pillow-heif (HEIC/HEIF, AVIF)" + else + optional "Python package pillow-heif (pip name: pillow-heif): HEIC and AVIF samples, if ffmpeg cannot do them" + fi + else + need "Python package Pillow (pip name: Pillow), for ${PYTHON_FOUND[*]}" + fi +else + need "Python 3.8+" +fi + +if command -v ffmpeg >/dev/null 2>&1; then + ok "ffmpeg: $(command -v ffmpeg)" + enc="$(ffmpeg -hide_banner -encoders 2>/dev/null)" + for e in libx264 libx265 libvpx-vp9 libaom-av1 libsvtav1; do + if grep -q " $e " <<<"$enc"; then ok "ffmpeg encoder $e"; else optional "ffmpeg encoder $e (build of ffmpeg without it)"; fi + done +else + optional "ffmpeg: video samples (MP4/HEVC/MOV/WebM) and, without pillow-heif, AVIF are skipped" +fi + +exit $missing diff --git a/.agents/skills/line-test-plan/scripts/make-samples.py b/.agents/skills/line-test-plan/scripts/make-samples.py new file mode 100644 index 00000000..5577b914 --- /dev/null +++ b/.agents/skills/line-test-plan/scripts/make-samples.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Generate labeled sample media for a LINE bridge test plan. + +Every file carries its own name as big text, a frame counter on animated ones, and +coloured corner markers (red top-left, green top-right, blue bottom-left, yellow +bottom-right) so a screenshot shows how the image was cropped, resized or flipped. +No real photos or personal data are used. + +Works on macOS, Linux and Windows. Needs Python 3.8+ and Pillow. ffmpeg is optional +(video, and AVIF/HEIC when pillow-heif is missing). A format that cannot be encoded +here is skipped and listed at the end; nothing is faked. + +Usage: + python3 make-samples.py OUT_DIR [--kinds image,animated,video,shape,ladder] [--only 01,10,13] + +`ladder` is the size ladder: animated GIFs of about 5, 10, 20 and 27 MB (22 to 27 MB +of random noise in total is too heavy for a default run), so it is only built when asked +for with --kinds ladder or --only. The other kinds are the default. + +Writes OUT_DIR/-. and OUT_DIR/MANIFEST.txt (file name, label, size). +Copy the labels into the `## Test data` section of PLAN.md. +""" + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile + +try: + from PIL import Image, ImageDraw, ImageFont +except ImportError: + sys.exit("Pillow is missing (pip name: Pillow). Run line-test-plan-preflight.sh.") + +try: # optional: HEIC/AVIF through Pillow + import pillow_heif + + pillow_heif.register_heif_opener() + try: + pillow_heif.register_avif_opener() + except AttributeError: + pass + HAVE_HEIF = True +except ImportError: + HAVE_HEIF = False + +FFMPEG = shutil.which("ffmpeg") +MARKERS = [((220, 40, 40), (0, 0)), ((40, 170, 60), (1, 0)), ((40, 80, 220), (0, 1)), ((240, 200, 30), (1, 1))] + +# (number, name, extension, kind, width, height) +SPECS = [ + (1, "static", "jpg", "image", 800, 800), + (2, "static-opaque", "png", "image", 800, 800), + (3, "static-transparent", "png", "image", 800, 800), + (4, "static", "webp", "image", 800, 800), + (5, "static", "heic", "image", 800, 800), + (6, "static", "avif", "image", 800, 800), + (7, "static", "bmp", "image", 800, 800), + (8, "static", "tiff", "image", 800, 800), + (9, "static", "gif", "image", 800, 800), + (10, "animated", "gif", "animated", 480, 480), + (11, "animated", "webp", "animated", 480, 480), + (12, "animated", "apng", "animated", 480, 480), + (13, "h264", "mp4", "video", 640, 480), + (14, "hevc", "mp4", "video", 640, 480), + (15, "h264", "mov", "video", 640, 480), + (16, "vp9", "webm", "video", 640, 480), + (17, "wide-1280x480", "jpg", "shape", 1280, 480), + (18, "tall-480x1280", "jpg", "shape", 480, 1280), + (19, "large-4096", "jpg", "shape", 4096, 4096), + (20, "tiny-64x64", "png", "shape", 64, 64), + (21, "animated-large", "gif", "animated", 1080, 1080), + (22, "animated-transparent", "gif", "animated", 480, 480), + (23, "animated-tiny", "gif", "animated", 32, 32), + (24, "animated-27mb", "gif", "ladder", 480, 480), + (25, "animated-wide", "gif", "animated", 960, 360), + (26, "animated-tall", "gif", "animated", 360, 960), + (27, "animated-square", "gif", "animated", 600, 600), + (28, "animated-5mb", "gif", "ladder", 480, 480), + (29, "animated-10mb", "gif", "ladder", 480, 480), + (30, "animated-20mb", "gif", "ladder", 480, 480), +] + +# Size ladder targets in megabytes (decimal). The file lands a little under the target, +# so "20mb" stays below a 20 MB limit measured in either MB or MiB. +LADDER_MB = {24: 27, 28: 5, 29: 10, 30: 20} + + +def font(size): + for name in ("DejaVuSans-Bold.ttf", "Arial Bold.ttf", "arialbd.ttf", "Helvetica.ttc"): + try: + return ImageFont.truetype(name, size) + except OSError: + continue + try: + return ImageFont.load_default(size=size) # Pillow >= 10.1 + except TypeError: + return ImageFont.load_default() + + +def frame(w, h, label, index=None, total=None, transparent=False): + """One labeled frame. Animated frames lighten the background and show N/total.""" + short = min(w, h) + if transparent: + img = Image.new("RGBA", (w, h), (0, 0, 0, 0)) + else: + t = 0 if index is None else index / max(total, 1) + grey = int(50 + 110 * t) # grey keeps the four corner markers visible on every frame + img = Image.new("RGB", (w, h), (grey, grey, grey)) + d = ImageDraw.Draw(img) + m = max(short // 8, 3) + for color, (cx, cy) in MARKERS: + x0 = 0 if cx == 0 else w - m + y0 = 0 if cy == 0 else h - m + d.rectangle([x0, y0, x0 + m - 1, y0 + m - 1], fill=color) + text = label if index is None else f"{label}\n{index + 1}/{total}" + size = max(short // 12, 8) + f = font(size) + while size > 6 and max(d.textlength(line, font=f) for line in text.split("\n")) > w - 2 * m - 4: + size -= 2 + f = font(size) + d.multiline_text((w / 2, h / 2), text, font=f, fill=(255, 255, 255, 255), anchor="mm", align="center", + stroke_width=max(size // 12, 1), stroke_fill=(0, 0, 0, 255)) + return img + + +def frames(w, h, label, count=12): + return [frame(w, h, label, i, count) for i in range(count)] + + +def noisy_frame(w, h, label, index): + """A frame whose top and bottom thirds are random noise (it barely compresses, so the + file size is predictable) and whose middle band and corners keep the label and markers.""" + base = frame(w, h, label, index, 1000) + # 6 bits of noise per pixel leaves most of the 256 colours to the label and markers + noise = Image.frombytes("L", (w, h), bytes(b & 0xFC for b in os.urandom(w * h))).convert("RGB") + band = (0, h // 3, w, 2 * h // 3) + noise.paste(base.crop(band), band[:2]) + m = max(min(w, h) // 8, 3) + for _, (cx, cy) in MARKERS: + box = (0 if cx == 0 else w - m, 0 if cy == 0 else h - m) + noise.paste(base.crop(box + (box[0] + m, box[1] + m)), box) + return noise.convert("P", palette=Image.ADAPTIVE) + + +def save_gif(path, fr): + fr[0].save(path, save_all=True, append_images=fr[1:], duration=100, loop=0) + return os.path.getsize(path) + + +def build_ladder(out, w, h, label, megabytes): + """Animated GIF of about `megabytes` MB: add noisy frames until just under the target.""" + target = int(megabytes * 1_000_000 * 0.985) + fr = [noisy_frame(w, h, label, 0)] + per_frame = save_gif(out, fr) + count = max(2, target // max(per_frame, 1)) + for _ in range(8): + while len(fr) < count: + fr.append(noisy_frame(w, h, label, len(fr))) + del fr[count:] + size = save_gif(out, fr) + if size <= target and size >= target * 0.97: + return + count = max(2, int(count * target / size)) + while size > target and len(fr) > 2: # settle just under the target + fr.pop() + size = save_gif(out, fr) + + +def ffmpeg(args): + r = subprocess.run([FFMPEG, "-hide_banner", "-loglevel", "error", "-y"] + args, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True) + return r.returncode == 0, r.stderr.strip() + + +def via_ffmpeg_image(png_path, out, codec_args): + return ffmpeg(["-i", png_path] + codec_args + [out]) + + +def make_video(out, w, h, label, vcodec_args): + tmp = tempfile.mkdtemp() + try: + for i, im in enumerate(frames(w, h, label, 30)): + im.convert("RGB").save(os.path.join(tmp, f"f{i:03d}.png")) + return ffmpeg(["-framerate", "10", "-i", os.path.join(tmp, "f%03d.png"), "-pix_fmt", "yuv420p"] + + vcodec_args + [out]) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def build(num, name, ext, kind, w, h, out_dir): + """Returns (ok, note). note explains a skip.""" + label = f"{num:02d}-{name}" + out = os.path.join(out_dir, f"{label}.{ext}") + if min(w, h) < 64: + label = f"{num:02d}" # too small for the full name; MANIFEST.txt maps the number to the file + transparent = name == "static-transparent" + try: + if kind == "ladder": + build_ladder(out, w, h, label, LADDER_MB[num]) + return True, "" + if kind == "animated": + if name == "animated-transparent": + fr = [frame(w, h, label, i, 12, transparent=True) for i in range(12)] + pal = [] + for f in fr: # index 255 is the transparent colour; disposal 2 clears each frame + p = f.convert("RGB").convert("P", palette=Image.ADAPTIVE, colors=255) + p.paste(255, f.getchannel("A").point(lambda a: 255 if a < 128 else 0)) + p.info["transparency"] = 255 + pal.append(p) + pal[0].save(out, save_all=True, append_images=pal[1:], duration=100, loop=0, + transparency=255, disposal=2) + return True, "" + fr = frames(w, h, label) + if ext == "gif": + fr = [f.convert("P", palette=Image.ADAPTIVE) for f in fr] + fr[0].save(out, save_all=True, append_images=fr[1:], duration=100, loop=0) + elif ext in ("webp", "apng"): + fmt = "WEBP" if ext == "webp" else "PNG" + fr[0].save(out, format=fmt, save_all=True, append_images=fr[1:], duration=100, loop=0) + return True, "" + if kind == "video": + args = {"h264": ["-c:v", "libx264"], "hevc": ["-c:v", "libx265", "-tag:v", "hvc1"], + "vp9": ["-c:v", "libvpx-vp9"]}[name] + if not FFMPEG: + return False, "needs ffmpeg" + ok, err = make_video(out, w, h, label, args) + return ok, "" if ok else f"ffmpeg cannot encode {name}: {err.splitlines()[-1] if err else 'unknown'}" + # still images and shapes + im = frame(w, h, label, transparent=transparent) + if ext == "jpg": + im.convert("RGB").save(out, quality=90) + elif ext == "png": + im.save(out) + elif ext == "bmp": + im.convert("RGB").save(out) + elif ext == "tiff": + im.convert("RGB").save(out) + elif ext == "gif": + im.convert("P", palette=Image.ADAPTIVE).save(out) + elif ext == "webp": + im.convert("RGB").save(out, quality=90) + elif ext in ("heic", "avif"): + if HAVE_HEIF: + im.convert("RGB").save(out, format="HEIF" if ext == "heic" else "AVIF") + elif FFMPEG: + tmp = out + ".src.png" + im.convert("RGB").save(tmp) + codec = ["-c:v", "libx265", "-tag:v", "hvc1", "-frames:v", "1"] if ext == "heic" else \ + ["-c:v", "libaom-av1", "-still-picture", "1", "-frames:v", "1"] + ok, err = via_ffmpeg_image(tmp, out, codec) + os.remove(tmp) + if not ok: + return False, f"{ext.upper()} needs pillow-heif; ffmpeg could not encode it" + else: + return False, f"{ext.upper()} needs pillow-heif or ffmpeg" + return True, "" + except Exception as e: # a codec missing in this Pillow build, for example + if os.path.exists(out): + os.remove(out) + return False, f"{type(e).__name__}: {e}" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("out_dir") + ap.add_argument("--kinds", default="image,animated,video,shape", + help="comma list of: image, animated, video, shape, ladder " + "(default: all but ladder, the 5-27 MB size ladder)") + ap.add_argument("--only", default="", help="comma list of sample numbers, e.g. 1,10,13") + a = ap.parse_args() + + # --only names files outright, whatever --kinds says (the ladder is only built that way or with --kinds ladder) + kinds = {k.strip() for k in a.kinds.split(",") if k.strip()} + only = {int(x) for x in a.only.split(",") if x.strip()} + os.makedirs(a.out_dir, exist_ok=True) + + made, skipped = [], [] + # Build into a scratch folder and move a file in only when it succeeded: a failed + # encoder (ffmpeg can leave a partial file) never replaces or leaves a file in out_dir, + # and a sample from an earlier run survives a rerun that cannot rebuild it. + scratch = tempfile.mkdtemp(prefix=".build-", dir=a.out_dir) + try: + for spec in SPECS: + num, name, ext, kind, w, h = spec + if only: + if num not in only: + continue + elif kind not in kinds: + continue + ok, note = build(*spec, scratch) + fname = f"{num:02d}-{name}.{ext}" + path = os.path.join(a.out_dir, fname) + if ok: + os.replace(os.path.join(scratch, fname), path) + (made if ok else skipped).append((fname, f"{num:02d}-{name}", note, path)) + finally: + shutil.rmtree(scratch, ignore_errors=True) + + # Rebuild the manifest from every sample on disk, so a partial rerun + # (for example --kinds video after images) keeps the earlier labels. + with open(os.path.join(a.out_dir, "MANIFEST.txt"), "w", encoding="utf-8") as f: + f.write("file\tlabel\tbytes\n") + for num, name, ext, _, _, _ in SPECS: + fname = f"{num:02d}-{name}.{ext}" + path = os.path.join(a.out_dir, fname) + if os.path.isfile(path): + f.write(f"{fname}\t{num:02d}-{name}\t{os.path.getsize(path)}\n") + + print(f"made {len(made)} sample(s) in {a.out_dir}") + for fname, _, _, path in made: + print(f" {fname} {os.path.getsize(path)} bytes") + if skipped: + print(f"skipped {len(skipped)}:") + for fname, _, note, _ in skipped: + print(f" {fname} {note}") + print("Skipped formats cannot be tested with generated files; say so in the plan or install the missing tool.") + print("Labels for PLAN.md `## Test data`: see MANIFEST.txt") + + +if __name__ == "__main__": + main() From 46ae1ac3f200c5105911f0ccae93499f7076f9d5 Mon Sep 17 00:00:00 2001 From: Caio Lins Date: Tue, 27 Oct 2026 23:57:22 +0900 Subject: [PATCH 3/3] tooling: add the line-evidence skill Turns a feature's test screenshots into PR-ready evidence: collects them from local folders, builds a local contact sheet to pick from, finds names, faces, phone numbers, emails and MIDs with on-device OCR (Apple Vision on macOS, tesseract elsewhere), covers everything outside the test plan's test data with solid boxes, and writes approved, metadata-free PNGs plus a manifest. Raw screenshots never leave the machine. Co-Authored-By: Claude Opus 5.5 --- .agents/skills/line-evidence/SKILL.md | 142 ++++++++++ .../skills/line-evidence/scripts/collect.py | 185 +++++++++++++ .../line-evidence/scripts/contact-sheet.py | 81 ++++++ .../skills/line-evidence/scripts/detect.py | 242 +++++++++++++++++ .../line-evidence/scripts/evidence_lib.py | 231 ++++++++++++++++ .../scripts/line-evidence-preflight.sh | 89 +++++++ .../skills/line-evidence/scripts/manifest.py | 209 +++++++++++++++ .../skills/line-evidence/scripts/redact.py | 250 ++++++++++++++++++ .../line-evidence/scripts/vision-detect.swift | 68 +++++ 9 files changed, 1497 insertions(+) create mode 100644 .agents/skills/line-evidence/SKILL.md create mode 100755 .agents/skills/line-evidence/scripts/collect.py create mode 100755 .agents/skills/line-evidence/scripts/contact-sheet.py create mode 100755 .agents/skills/line-evidence/scripts/detect.py create mode 100755 .agents/skills/line-evidence/scripts/evidence_lib.py create mode 100755 .agents/skills/line-evidence/scripts/line-evidence-preflight.sh create mode 100755 .agents/skills/line-evidence/scripts/manifest.py create mode 100755 .agents/skills/line-evidence/scripts/redact.py create mode 100644 .agents/skills/line-evidence/scripts/vision-detect.swift diff --git a/.agents/skills/line-evidence/SKILL.md b/.agents/skills/line-evidence/SKILL.md new file mode 100644 index 00000000..c9aa07ed --- /dev/null +++ b/.agents/skills/line-evidence/SKILL.md @@ -0,0 +1,142 @@ +--- +name: line-evidence +description: Turns the screenshots from a beeper/line feature test into PR-ready evidence with personal information covered. Collects screenshots from local folders, builds a local contact sheet to pick from, finds text, phone numbers, emails, LINE MIDs and faces with on-device OCR, covers everything not on the test-data allowlist with solid boxes, previews before/after for approval, and writes metadata-free PNGs plus a manifest. Use after testing a feature on the Chrome extension, Beeper Desktop, Beeper mobile or LINE iOS, or when asked to redact, collect or prepare screenshots or test evidence for a PR. +--- + +# LINE Evidence + +Turns a feature's test screenshots into evidence for a PR, with every name, +avatar, face, phone number, email and MID covered. Run the commands below +from the repository root. The scripts are Python 3.8+ with Pillow and run the +same on macOS, Linux and Windows (Git Bash). + +## Hard rule: raw images stay on this machine + +Raw screenshots, the contact sheet, previews and any image not yet approved +show real contacts and chats. They never leave the machine: no artifacts, no +PR or issue uploads, no chat attachments to other services, no pastebins or +image hosts, no commits. Open them only from disk. Looking at them yourself +to find PII is part of this skill; sending them anywhere else is not. Only +approved files in `evidence/redacted/` may be uploaded later, by the user, +taking the list from `manifest.py eligible`. + +## Layout + +Shared with `line-test-plan`: the feature folder is `test//`, +and its `PLAN.md` has step IDs like `B5` (`^[A-Z][0-9]+$`), a `Platform` and a +`Phase` (`before` | `capture` | `after`) column, and a `## Test data` section +of dummy strings. + +``` +test// + PLAN.md + evidence/ + raw/ never shared (git-ignored by its own .gitignore) + index.json collected files, their source and time + contact-sheet.html + previews/, review.html + boxes/.json boxes per raw image, e.g. 20261025-044601.png.json; no hidden text + redacted/--.png + manifest.json step -> files, platform, phase, approved, in_pr +``` + +Below, `F=test/` and `S=.agents/skills/line-evidence/scripts`. +Use `python3`, or the Python the preflight reported. + +## Workflow + +1. **Check dependencies.** Run `bash $S/line-evidence-preflight.sh`. It prints + `OK`, `MISSING` (required) and `OPTIONAL` (works, with a gap) lines and no + install commands. Pick commands that fit the OS and its package managers + (for example ` -m pip install --user Pillow`, Homebrew, apt, + winget), show them, and install only after the user confirms. On macOS, + Apple Vision does OCR and faces. Elsewhere tesseract does OCR, and faces need + `opencv-python-headless`; without it, tell the user faces are found only by + looking, and look extra carefully. + +2. **Collect.** Ask where the screenshots are, suggesting `~/Downloads` and + `~/Desktop` (AirDrop, downloads and shares land there); accept several + paths. Ask for the time window of the test session, or let the script take + it from the date in `PLAN.md` (that day 00:00 until now) and tell the user + which window it used. + ```bash + python3 $S/collect.py --feature $F --since "2026-10-25 03:00" ~/Downloads ~/Desktop + python3 $S/collect.py --feature $F --add /path/to/agent-screenshot.png # screenshots you took yourself + ``` + It keeps only images in the window, skips duplicates by SHA-256, turns + HEIC and rotated JPEGs into upright PNGs, and copies them to + `evidence/raw/`. Use `--dry-run` first when the folders are large. + +3. **Pick.** Look at the collected images, write your guesses as + `{"": {"platform", "step", "phase"}}` to a scratch file, and build + the contact sheet: + ```bash + python3 $S/contact-sheet.py --feature $F --guesses /tmp/guesses.json + ``` + Give the user the path to `evidence/raw/contact-sheet.html` to open from + disk. **Never publish it**: no artifact, no upload, no file card to another + service. The user answers like `3, 7, 12; 7 is B5 after`. Record each pick + (platform and phase default to the step's row in `PLAN.md`): + ```bash + python3 $S/manifest.py --feature $F pick --step B5 --phase after 20261025-044601.png + ``` + +4. **Find PII** in each picked image: + ```bash + python3 $S/detect.py --feature $F 20261025-044601.png [--allow "LSON Group" ...] + ``` + This runs on-device OCR (Vision on macOS, else tesseract) and face + detection, and checks phone numbers, emails and LINE MIDs + (`[UCR][0-9a-f]{32}`). Every word is covered unless it is in `PLAN.md`'s + `## Test data`, an `--allow` string (kept in the boxes file for re-runs), or + the built-in generic UI list in `evidence_lib.py`. Then **look at each image + yourself**: OCR misses avatars, photos of people, stylized names, QR codes, + phone status bars and text inside images. Add a box for each of those (step 5). + +5. **Preview and adjust.** + ```bash + python3 $S/redact.py --feature $F preview 20261025-044601.png + ``` + This writes `evidence/raw/previews/.png` (left: numbered boxes, red + covered, green visible, blue added; right: the result) and + `evidence/raw/review.html`, and prints the numbered box list. Look at the + preview yourself, then give the user the `review.html` path (local only). + Apply what they say by editing `evidence/boxes/.json`: + - "unhide 4": add box 4's `id` to `unhide` (for a red box). + - "hide 9": add box 9's `id` to `hide` (for a green box). + - "also hide the top bar": append `{"x", "y", "w", "h", "label"}` in pixels + to `add`, measured from the image. + Re-run `preview` after each change. Boxes are solid fills, never blur. + Re-running `detect.py` keeps `add`, `hide`, `unhide` and `allow`. + +6. **Approve and write.** Only after the user says the preview is fine: + ```bash + python3 $S/redact.py --feature $F approve 20261025-044601.png + python3 $S/redact.py --feature $F final 20261025-044601.png + ``` + `approve` stores a hash of the raw image's bytes and the covered boxes. + Replacing the screenshot or changing any box voids it, and the scripts then + delete the redacted file made under the old approval, so `redacted/` only + ever holds currently approved output. + `final` needs the approval and a manifest entry, re-encodes to PNG from bare + pixels (no EXIF, GPS, text chunks or ICC profile), writes + `evidence/redacted/--.png`, OCRs the result again and + warns about readable text, and marks the entry approved in + `manifest.json`. Open each output yourself before calling it done. + +7. **Report.** Run `python3 $S/manifest.py --feature $F show` and list the + redacted files per step. `manifest.py eligible` prints only files that are + approved, unchanged on disk and marked for the PR: that is the upload list. `manifest.py set --in-pr no ` leaves a + file out of the PR. Tell the user that `evidence/raw/` still holds the + unredacted originals and ask whether to delete it once the PR is merged. + +## Notes + +- The generic UI allowlist is deliberately short. Feature strings belong in + `PLAN.md`'s `## Test data`, not in the script. Plans older than + `line-test-plan` have no such section: pass their dummy strings with + `--allow`. +- Hidden text is never written to `boxes/*.json`; only its kind and length. + Visible (allowlisted) text is. +- A screenshot with nothing to show after redaction is not evidence. Say so, + and suggest retaking it with test data instead of real chats. diff --git a/.agents/skills/line-evidence/scripts/collect.py b/.agents/skills/line-evidence/scripts/collect.py new file mode 100755 index 00000000..043054c7 --- /dev/null +++ b/.agents/skills/line-evidence/scripts/collect.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Collect a feature's test screenshots into test//evidence/raw/. + +Usage: + collect.py --feature test/ [--since T] [--until T] [--add FILE ...] [--dry-run] PATH... + +PATH is a folder (not searched recursively) or an image file. Only images +taken inside the time window are kept: --since/--until accept +"YYYY-MM-DD", "YYYY-MM-DD HH:MM" or ISO 8601. Without --since the window +starts at 00:00 on PLAN.md's date (or the day PLAN.md was last changed) and +runs until now. --add files (screenshots the agent took itself) skip the +window check. An image's time is its EXIF DateTimeOriginal, else its mtime. + +Duplicates (same SHA-256, also against earlier runs) are skipped. HEIC/HEIF +becomes PNG, and JPEGs with an EXIF rotation are turned upright and saved as +PNG, so box coordinates always match the pixels. Copies are named +[-n]. and listed in raw/index.json with their source. +""" + +import argparse +import hashlib +import os +import shutil +import subprocess +import sys +from datetime import datetime + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from evidence_lib import IMAGE_EXTS, Feature, die, load_json, parse_plan, save_json # noqa: E402 + +try: + from PIL import Image, ImageOps +except ImportError: + die("Pillow is missing; run the preflight") + + +def parse_time(s): + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d"): + try: + return datetime.strptime(s, fmt) + except ValueError: + pass + try: + t = datetime.fromisoformat(s) + return t.astimezone().replace(tzinfo=None) if t.tzinfo else t + except ValueError: + die(f"cannot read time {s!r}") + + +def taken_at(path): + """EXIF DateTimeOriginal when the file has one, else the mtime.""" + if os.path.splitext(path)[1].lower() in (".jpg", ".jpeg", ".tif", ".tiff", ".webp", ".png"): + try: + with Image.open(path) as im: + exif = im.getexif() + raw = exif.get_ifd(0x8769).get(0x9003) or exif.get(0x0132) + if raw: + return datetime.strptime(str(raw).strip("\x00 "), "%Y:%m:%d %H:%M:%S") + except Exception: + pass + return datetime.fromtimestamp(os.path.getmtime(path)) + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def heic_to_png(src, dst): + try: + import pillow_heif + + pillow_heif.register_heif_opener() + with Image.open(src) as im: + ImageOps.exif_transpose(im).save(dst, "PNG") + return + except ImportError: + pass + if shutil.which("sips"): + subprocess.run(["sips", "-s", "format", "png", src, "--out", dst], check=True, capture_output=True) + elif shutil.which("heif-convert"): + subprocess.run(["heif-convert", src, dst], check=True, capture_output=True) + else: + die(f"cannot convert HEIC ({src}): no pillow-heif, sips or heif-convert; run the preflight") + + +def candidates(paths): + for p in paths: + p = os.path.expanduser(p) + if os.path.isdir(p): + for name in sorted(os.listdir(p)): + full = os.path.join(p, name) + if os.path.isfile(full) and os.path.splitext(name)[1].lower() in IMAGE_EXTS: + yield full + elif os.path.isfile(p): + yield p + else: + print(f"skip: not found: {p}", file=sys.stderr) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--feature", required=True) + ap.add_argument("--plan") + ap.add_argument("--since") + ap.add_argument("--until") + ap.add_argument("--add", nargs="*", default=[], help="agent screenshots; no time check") + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("paths", nargs="*") + a = ap.parse_args() + feat = Feature(a.feature, a.plan) + + if a.since: + since = parse_time(a.since) + else: + _, _, date = parse_plan(feat.plan) + if date: + since = parse_time(date) + elif os.path.exists(feat.plan): + since = datetime.fromtimestamp(os.path.getmtime(feat.plan)).replace(hour=0, minute=0, second=0) + else: + die("no --since and no PLAN.md to infer the window from; ask the user for the time window") + until = parse_time(a.until) if a.until else datetime.now() + print(f"window: {since:%Y-%m-%d %H:%M} .. {until:%Y-%m-%d %H:%M}") + + index = feat.load_index() + seen = {e["sha256"] for e in index} + names = {e["file"] for e in index} + picked = [] + for src, forced in [(p, False) for p in candidates(a.paths)] + [(p, True) for p in candidates(a.add)]: + t = taken_at(src) + if not forced and not (since <= t <= until): + continue + digest = sha256(src) + if digest in seen: + print(f"dup: {src}") + continue + seen.add(digest) + picked.append((t, src, digest, forced)) + + picked.sort() + if not a.dry_run and picked: + feat.make_raw() + for t, src, digest, forced in picked: + ext = os.path.splitext(src)[1].lower() + convert = ext in (".heic", ".heif") + if not convert and ext in (".jpg", ".jpeg"): + with Image.open(src) as im: + convert = im.getexif().get(0x0112, 1) != 1 + out_ext = ".png" if convert else (".jpg" if ext == ".jpeg" else ext) + base, n = t.strftime("%Y%m%d-%H%M%S"), 1 + name = base + out_ext + while name in names: + n += 1 + name = f"{base}-{n}{out_ext}" + names.add(name) + print(f"add: {name} <- {src}") + if a.dry_run: + continue + dst = os.path.join(feat.raw, name) + if ext in (".heic", ".heif"): + heic_to_png(src, dst) + elif convert: + with Image.open(src) as im: + ImageOps.exif_transpose(im).save(dst, "PNG") + else: + shutil.copyfile(src, dst) + os.chmod(dst, 0o600) + with Image.open(dst) as im: + size = im.size + index.append({"file": name, "source": os.path.abspath(src), "sha256": digest, + "taken": t.isoformat(timespec="seconds"), "width": size[0], "height": size[1], + "agent": forced}) + + if not a.dry_run and picked: + index.sort(key=lambda e: (e["taken"], e["file"])) + save_json(feat.index, index) + print(f"{len(picked)} new image(s), {len(index)} in {feat.raw}" + (" (dry run)" if a.dry_run else "")) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/line-evidence/scripts/contact-sheet.py b/.agents/skills/line-evidence/scripts/contact-sheet.py new file mode 100755 index 00000000..4780450d --- /dev/null +++ b/.agents/skills/line-evidence/scripts/contact-sheet.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Build raw/contact-sheet.html: numbered thumbnails of the collected screenshots. + +Usage: + contact-sheet.py --feature test/ [--guesses guesses.json] + +Numbers follow raw/index.json (oldest first), so "3, 7, 12" from the user maps +back to files. Each card shows a guessed platform from the image shape and +file name. --guesses is a JSON object the agent writes after looking at the +images, {"3": {"platform": "LINE iOS", "step": "B5", "phase": "after"}, ...}; +it replaces the shape guess. The page uses relative links to the raw files +and is meant to be opened from disk only: it shows unredacted PII. +""" + +import argparse +import html +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from evidence_lib import Feature, die, load_json # noqa: E402 + + +def shape_guess(e): + w, h = e.get("width") or 0, e.get("height") or 0 + src = os.path.basename(e.get("source", "")) + if h and h / max(w, 1) > 1.9: + return "phone (LINE iOS or Beeper mobile)" + if src.upper().startswith("IMG_"): + return "phone" + if w and h and w <= 900 and h <= 1100: + return "desktop window (Chrome extension?)" + return "desktop (Beeper Desktop or Chrome)" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--feature", required=True) + ap.add_argument("--guesses") + a = ap.parse_args() + feat = Feature(a.feature) + index = feat.load_index() + if not index: + die(f"nothing collected in {feat.raw}; run collect.py first") + guesses = load_json(a.guesses, {}) if a.guesses else {} + + cards = [] + for n, e in enumerate(index, 1): + g = guesses.get(str(n), {}) + platform = g.get("platform") or shape_guess(e) + step = " ".join(x for x in (g.get("step"), g.get("phase")) if x) or "?" + tag = " · agent" if e.get("agent") else "" + src = html.escape(e["file"]) + cards.append( + f"
" + f"
{n} {html.escape(step)}
" + f"{html.escape(platform)}
{html.escape(e['taken'].replace('T', ' '))} · " + f"{e.get('width')}×{e.get('height')}{tag}
") + + page = f"""Contact sheet: {html.escape(feat.slug)} + +

Local file with unredacted screenshots. Do not upload, publish or share it.

+

{html.escape(feat.slug)}: {len(index)} screenshots

+

Reply with the numbers to use and what they show, for example: 3, 7, 12; 7 is B5 after.

+
{''.join(cards)}
""" + out = os.path.join(feat.raw, "contact-sheet.html") + with open(out, "w", encoding="utf-8") as f: + f.write(page) + os.chmod(out, 0o600) + for n, e in enumerate(index, 1): + print(f"{n:3} {e['file']} {guesses.get(str(n), {}).get('platform') or shape_guess(e)}") + print(f"contact sheet: {out}") + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/line-evidence/scripts/detect.py b/.agents/skills/line-evidence/scripts/detect.py new file mode 100755 index 00000000..9654baac --- /dev/null +++ b/.agents/skills/line-evidence/scripts/detect.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Find text, PII and faces in raw screenshots and write evidence/boxes/.json. + +Usage: + detect.py --feature test/ [--plan PLAN.md] [--allow TEXT ...] [--engine auto|vision|tesseract] IMAGE... + +IMAGE is a file in evidence/raw/ (name or path). Everything runs on the device: +Apple Vision on macOS (a small Swift program, compiled once into a cache +folder), tesseract elsewhere. Faces come from Vision, or OpenCV when it is +installed; otherwise the script says faces were not checked. + +Each OCR word is hidden unless it is part of a PLAN.md `## Test data` string, +an --allow string, or the built-in generic UI list. Phone numbers, emails and +LINE MIDs are always hidden unless the match is itself test data. Hidden +words next to each other on a line become one box. + +A re-run replaces only the "auto" and "kept" lists. "add", "hide" and +"unhide" (your edits) stay, and box IDs are stable, so the result repeats. +The JSON never stores hidden text, only its kind and length. +""" + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +import unicodedata + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import manifest # noqa: E402 +from evidence_lib import PII, SAFE_WORD, Feature, die, load_json, norm, parse_plan, save_json, ui_phrases, words # noqa: E402 + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def cache_dir(): + base = os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache") + return os.path.join(base, "line-evidence") + + +def vision_binary(): + """Compile vision-detect.swift once per source version; None when not on macOS.""" + if sys.platform != "darwin" or not shutil.which("swiftc"): + return None + src = os.path.join(HERE, "vision-detect.swift") + tag = hashlib.sha256(open(src, "rb").read()).hexdigest()[:12] + out = os.path.join(cache_dir(), f"vision-detect-{tag}") + if not os.path.exists(out): + os.makedirs(cache_dir(), exist_ok=True) + r = subprocess.run(["swiftc", "-O", "-o", out, src], capture_output=True, text=True) + if r.returncode: + print(f"warning: cannot compile Vision helper, falling back:\n{r.stderr[-800:]}", file=sys.stderr) + return None + return out + + +def run_vision(binary, image): + r = subprocess.run([binary, image], capture_output=True, text=True) + if r.returncode: + die(f"Vision failed on {image}: {r.stderr.strip()}") + return json.loads(r.stdout) + + +def run_tesseract(image): + from PIL import Image + + langs = subprocess.run(["tesseract", "--list-langs"], capture_output=True, text=True).stdout.split() + lang = "+".join(l for l in ("eng", "jpn") if l in langs) or "eng" + r = subprocess.run(["tesseract", image, "-", "-l", lang, "tsv"], capture_output=True, text=True) + if r.returncode: + die(f"tesseract failed on {image}: {r.stderr.strip()}") + out, lines = [], {} + for row in r.stdout.splitlines()[1:]: + f = row.split("\t") + if len(f) < 12 or f[0] != "5" or not f[11].strip() or float(f[10]) < 0: + continue + key = (f[1], f[2], f[3], f[4]) + out.append({"text": f[11], "line": lines.setdefault(key, len(lines)), "conf": float(f[10]) / 100, + "x": int(f[6]), "y": int(f[7]), "w": int(f[8]), "h": int(f[9])}) + with Image.open(image) as im: + w, h = im.size + return {"width": w, "height": h, "words": out, "faces": opencv_faces(image)} + + +def opencv_faces(image): + """Faces with OpenCV's bundled Haar cascade, or None when OpenCV is not installed.""" + try: + import cv2 + except ImportError: + return None + img = cv2.imread(image) + if img is None: + return [] + cascade = cv2.CascadeClassifier(os.path.join(cv2.data.haarcascades, "haarcascade_frontalface_default.xml")) + found = cascade.detectMultiScale(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), 1.1, 5, minSize=(24, 24)) + return [{"x": int(x), "y": int(y), "w": int(w), "h": int(h)} for x, y, w, h in found] + + +def analyze(image, engine): + if engine in ("auto", "vision"): + binary = vision_binary() + if binary: + return "vision", run_vision(binary, image) + if engine == "vision": + die("Apple Vision is only available on macOS with swiftc (Xcode Command Line Tools)") + if not shutil.which("tesseract"): + die("no OCR engine: install tesseract (or use macOS with Vision); run the preflight") + return "tesseract", run_tesseract(image) + + +def phrase_spans(tokens, phrases): + """Indexes of tokens covered by any phrase (a list of normalized word lists).""" + covered = set() + for p in phrases: + n = len(p) + for i in range(len(tokens) - n + 1): + if tokens[i:i + n] == p: + covered.update(range(i, i + n)) + return covered + + +def box_id(kind, text, b): + key = f"{kind}|{norm(text)}|{b['x'] // 8}|{b['y'] // 8}|{b['w'] // 8}|{b['h'] // 8}" + return kind[0] + hashlib.sha1(key.encode()).hexdigest()[:7] + + +def union(bs): + x0, y0 = min(b["x"] for b in bs), min(b["y"] for b in bs) + x1, y1 = max(b["x"] + b["w"] for b in bs), max(b["y"] + b["h"] for b in bs) + return {"x": x0, "y": y0, "w": x1 - x0, "h": y1 - y0} + + +def classify(result, test_data, extra): + allowed = [words(s) for s in test_data + extra] + allowed = [p for p in allowed if p] + ui = [words(s) for s in ui_phrases()] + allowed_whole = {norm(s) for s in test_data + extra} + auto, kept = [], [] + + by_line = {} + for w in result["words"]: + by_line.setdefault(w["line"], []).append(w) + for line in by_line.values(): + line.sort(key=lambda w: w["x"]) + tokens = [norm(w["text"]) for w in line] + text = " ".join(w["text"] for w in line) + # Character offsets of each word in `text`, to map PII matches back to words. + offsets, pos = [], 0 + for w in line: + offsets.append((pos, pos + len(w["text"]))) + pos += len(w["text"]) + 1 + pii = {} + for kind, rx in PII: + for m in rx.finditer(text): + if norm(m.group()) in allowed_whole: + continue + for i, (s, e) in enumerate(offsets): + if s < m.end() and m.start() < e: + pii.setdefault(i, kind) + test = phrase_spans(tokens, allowed) + generic = phrase_spans(tokens, ui) + state = [] + for i, w in enumerate(line): + if i in pii: + state.append(("hide", pii[i])) + elif i in test: + state.append(("keep", "test data")) + elif i in generic or SAFE_WORD.match(unicodedata.normalize("NFKC", w["text"]).strip()): + state.append(("keep", "ui")) + else: + state.append(("hide", "text")) + i = 0 + while i < len(line): + verdict, why = state[i] + j = i + while j + 1 < len(line) and state[j + 1] == state[i]: + j += 1 + group = line[i:j + 1] + b = union(group) + joined = " ".join(w["text"] for w in group) + if verdict == "hide": + auto.append({"id": box_id(why, joined, b), "kind": why, **b, "chars": len(joined)}) + else: + kept.append({"id": box_id("keep", joined, b), "kind": why, **b, "text": joined}) + i = j + 1 + + for f in result.get("faces") or []: + auto.append({"id": box_id("face", "", f), "kind": "face", **{k: int(f[k]) for k in "xywh"}}) + return auto, kept + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--feature", required=True) + ap.add_argument("--plan") + ap.add_argument("--allow", nargs="*", default=[], help="extra strings that may stay visible") + ap.add_argument("--engine", default="auto", choices=["auto", "vision", "tesseract"]) + ap.add_argument("images", nargs="+") + a = ap.parse_args() + feat = Feature(a.feature, a.plan) + test_data, _, _ = parse_plan(feat.plan) + if not test_data and not a.allow: + print("warning: no `## Test data` in the plan and no --allow: only generic UI text stays visible", + file=sys.stderr) + + for name in a.images: + image = feat.raw_image(name) + engine, result = analyze(image, a.engine) + path = feat.box_path(image) + old = load_json(path, {}) + allow = a.allow or old.get("allow", []) + auto, kept = classify(result, test_data, allow) + data = { + "image": feat.rel(image), + "width": int(result["width"]), + "height": int(result["height"]), + "engine": engine, + "faces_checked": result.get("faces") is not None, + "allow": allow, + "auto": sorted(auto, key=lambda b: (b["y"], b["x"])), + "kept": sorted(kept, key=lambda b: (b["y"], b["x"])), + "add": old.get("add", []), + "hide": old.get("hide", []), + "unhide": old.get("unhide", []), + "approved": old.get("approved"), + } + save_json(path, data) + # New boxes can void the approval; delete outputs written under it. + manifest.sweep(feat) + kinds = {} + for b in auto: + kinds[b["kind"]] = kinds.get(b["kind"], 0) + 1 + summary = ", ".join(f"{v} {k}" for k, v in sorted(kinds.items())) or "nothing" + note = "" if data["faces_checked"] else " (faces NOT checked: look for faces and avatars by eye)" + print(f"{os.path.basename(image)}: hide {summary}; keep {len(kept)} [{engine}]{note}") + print(f" -> {path}") + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/line-evidence/scripts/evidence_lib.py b/.agents/skills/line-evidence/scripts/evidence_lib.py new file mode 100755 index 00000000..aa6b96e8 --- /dev/null +++ b/.agents/skills/line-evidence/scripts/evidence_lib.py @@ -0,0 +1,231 @@ +"""Shared helpers for the line-evidence scripts: paths, PLAN.md, allowlist.""" + +import hashlib +import json +import os +import re +import sys +import unicodedata +from datetime import datetime, timezone + +STEP = re.compile(r"^[A-Z][0-9]+$") +PHASES = ("before", "capture", "after") +IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".heic", ".heif", ".webp", ".gif", ".bmp", ".tif", ".tiff"} + +# Generic UI text that may stay visible. Phrases are matched word by word +# against OCR lines, so "Group settings" also keeps "Group" and "settings" +# only where they appear together. Keep this list to UI labels and words that +# are never names; feature-specific strings belong in PLAN.md's Test data. +UI_PHRASES = """ +Settings|Send|Cancel|Done|OK|Save|Edit|Delete|Remove|Back|Close|Search|Next|Confirm|Change|Apply|Retry +Chats|Chat|Friends|Home|Calls|News|Wallet|Add friends|Groups|Group|Members|Invite|Leave|Leave group +Profile|Profile photo|Group photo|Group name|Group settings|Edit group|Change photo|Remove photo +Choose photo|Select photo|Take photo|Upload|Uploading|Photos|Albums|Notes|Files|Links|Media|Images|Videos +Notifications|Mute|Unmute|Hide|Block|Report|Pin|Unpin|Reply|Forward|Copy|Share|Announcements +Today|Yesterday|Read|Delivered|Sent|Seen|Online|Offline|Typing|New message|Message|Messages +changed the group name to|changed the group profile photo|changed the group photo|removed the group photo +changed the group cover|set the group photo|joined the group|left the group|invited|joined|left +You|you|the|to|a|an|and|or|of|in|on|at|for|from|with|by|is|was|has|have|this|that|new|name|photo|cover +Beeper|LINE|Matrix|Inbox|Archive|Archived|Low priority|Favorites|Unread|All|Spaces|Bridges|Accounts +Letter Sealing|Chat settings|Background|Font size|Language|Help|About|Version|Log out|Logout|Login|Log in +Open|More|Info|Details|Members list|Admin|Owner|Admins|Others|Other|Show more|See all|View all +設定|送信|キャンセル|完了|保存|編集|削除|トーク|友だち|ホーム|グループ|メンバー|招待|退会|写真|アルバム|ノート +""" + +# Whole-word patterns that are safe to show: clock times, dates, counts, percent. +SAFE_WORD = re.compile( + r"^(\d{1,2}:\d{2}(:\d{2})?|[AaPp]\.?[Mm]\.?|\d{1,3}%?|\(\d{1,4}\)|\d{1,2}/\d{1,2}(/\d{2,4})?" + r"|\d{4}[-/.]\d{1,2}[-/.]\d{1,2}|[^\w]+)$" +) + +# PII patterns, applied to the whole OCR line so numbers split by spaces still match. +PII = [ + ("mid", re.compile(r"(?/ folder and its evidence layout.""" + + def __init__(self, path, plan=None): + self.dir = os.path.abspath(path) + if not os.path.isdir(self.dir): + die(f"feature folder not found: {path}") + self.slug = os.path.basename(self.dir) + self.evidence = os.path.join(self.dir, "evidence") + self.raw = os.path.join(self.evidence, "raw") + self.redacted = os.path.join(self.evidence, "redacted") + self.boxes = os.path.join(self.evidence, "boxes") + self.previews = os.path.join(self.raw, "previews") + self.manifest = os.path.join(self.evidence, "manifest.json") + self.index = os.path.join(self.raw, "index.json") + self.plan = plan or os.path.join(self.dir, "PLAN.md") + + def make_raw(self): + os.makedirs(self.raw, mode=0o700, exist_ok=True) + os.chmod(self.raw, 0o700) + # raw/ holds unredacted PII: keep git from ever picking it up. + ignore = os.path.join(self.raw, ".gitignore") + if not os.path.exists(ignore): + with open(ignore, "w") as f: + f.write("# Unredacted screenshots: never commit or share.\n*\n") + + # Keyed by the full file name, so a.png and a.jpg never share boxes or a preview. + def box_path(self, image): + return os.path.join(self.boxes, os.path.basename(image) + ".json") + + def preview_path(self, image): + return os.path.join(self.previews, os.path.basename(image) + ".png") + + def raw_image(self, name): + """Resolve an image argument (name, raw/ name or path) to a path inside raw/.""" + for cand in (name, os.path.join(self.raw, name), os.path.join(self.raw, os.path.basename(name))): + if os.path.isfile(cand) and os.path.dirname(os.path.abspath(cand)) == self.raw: + return os.path.abspath(cand) + die(f"not an image in {self.raw}: {name}") + + def rel(self, path): + return os.path.relpath(path, self.evidence).replace(os.sep, "/") + + def load_index(self): + return load_json(self.index, []) + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def numbered(data): + """Every box with its state, sorted top to bottom and numbered from 1.""" + out = [] + for b in data["auto"]: + out.append(dict(b, state="keep" if b["id"] in data.get("unhide", []) else "cover")) + for b in data["kept"]: + out.append(dict(b, state="cover" if b["id"] in data.get("hide", []) else "keep")) + for i, b in enumerate(data.get("add", [])): + out.append(dict(b, id=f"add{i}", kind="added", state="add")) + out.sort(key=lambda b: (b["y"], b["x"])) + for n, b in enumerate(out, 1): + b["n"] = n + return out + + +def covers(data): + return [b for b in numbered(data) if b["state"] in ("cover", "add")] + + +def rect(b, size): + """Padded rectangle, clamped to the image.""" + pad = 2 if b.get("state") == "add" else max(2, round(min(b["w"], b["h"]) * 0.15)) + W, H = size + return (max(0, b["x"] - pad), max(0, b["y"] - pad), min(W, b["x"] + b["w"] + pad), min(H, b["y"] + b["h"] + pad)) + + +def approval_hash(data, image): + """Hash of the raw image's bytes and the covered rectangles. + + An approval is only valid while this matches: replacing the screenshot or + changing any box voids it. + """ + rects = sorted(rect(b, (data["width"], data["height"])) for b in covers(data)) + return hashlib.sha256(json.dumps([sha256_file(image), rects]).encode()).hexdigest()[:16] + + +def is_approved(data, image): + return bool(data) and (data.get("approved") or {}).get("hash") == approval_hash(data, image) + + +def load_json(path, default): + if not os.path.exists(path): + return default + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def save_json(path, data): + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False, sort_keys=False) + f.write("\n") + os.replace(tmp, path) + + +def cells(line): + return [c.strip() for c in line.strip().strip("|").split("|")] + + +def parse_plan(path): + """Return (test_data, steps, date) from a PLAN.md. + + test_data: strings from `## Test data` bullets (backticked parts if any, + else the whole bullet). steps: {id: {"platform", "phase"}} from tables with + #, Platform and Phase columns. date: first YYYY-MM-DD near the top, or None. + A missing plan returns empty values, so older plans still work with --allow. + """ + if not path or not os.path.exists(path): + return [], {}, None + lines = open(path, encoding="utf-8").read().splitlines() + test_data, steps, header, section = [], {}, None, "" + date = None + for l in lines[:30]: + m = re.search(r"\b(20\d\d-\d\d-\d\d)\b", l) + if m: + date = m.group(1) + break + for l in lines: + if l.startswith("## "): + section, header = l.strip(), None + continue + if section == "## Test data" and l.lstrip().startswith(("- ", "* ")): + body = l.lstrip()[2:].strip() + ticks = re.findall(r"`([^`]+)`", body) + for t in ticks or [body]: + if t.strip() and t.strip().lower() != "none": + test_data.append(t.strip()) + if not l.lstrip().startswith("|"): + header = None if not l.strip() else header + continue + c = cells(l) + if c and c[0] == "#": + header = c + continue + if not header or set("".join(c)) <= set("-: ") or len(c) != len(header): + continue + row = dict(zip(header, c)) + if STEP.match(row.get("#", "")) and "Platform" in row and "Phase" in row: + steps[row["#"]] = {"platform": row["Platform"], "phase": row["Phase"]} + return test_data, steps, date + + +def slug(s): + s = unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode().lower() + return re.sub(r"[^a-z0-9]+", "-", s).strip("-") or "unknown" + + +def now_iso(): + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") diff --git a/.agents/skills/line-evidence/scripts/line-evidence-preflight.sh b/.agents/skills/line-evidence/scripts/line-evidence-preflight.sh new file mode 100755 index 00000000..5e457e95 --- /dev/null +++ b/.agents/skills/line-evidence/scripts/line-evidence-preflight.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Check that everything line-evidence needs is installed. +# Works on macOS, Linux and Windows (Git Bash / MSYS2). +# +# Usage: .agents/skills/line-evidence/scripts/line-evidence-preflight.sh +# +# Prints the detected OS, one OK line per dependency found and one MISSING +# line per dependency that is not. OPTIONAL lines are not required but leave +# a gap (for example no face detector). It deliberately does not say how to +# install anything: the agent picks the method that fits the OS and asks the +# user first. Exits 1 when anything required is missing. +# +# Override detection with PYTHON=/path/to/python. + +set -uo pipefail + +case "$(uname -s)" in + Darwin) OS=macos ;; + Linux) + if grep -qi microsoft /proc/version 2>/dev/null; then OS=wsl; else OS=linux; fi ;; + MINGW* | MSYS* | CYGWIN*) OS=windows ;; + *) OS=unknown ;; +esac + +PY=() +if [ -n "${PYTHON:-}" ]; then + "$PYTHON" -c "import sys; sys.exit(sys.version_info < (3, 8))" 2>/dev/null && PY=("$PYTHON") +else + for candidate in "python3" "python" "py -3"; do + # shellcheck disable=SC2206 # intentional split of "py -3" + cmd=($candidate) + if "${cmd[@]}" -c "import sys; sys.exit(sys.version_info < (3, 8))" 2>/dev/null; then + PY=("${cmd[@]}") + break + fi + done +fi + +missing=0 +ok() { echo "OK $1"; } +need() { echo "MISSING $1"; missing=1; } +opt() { echo "OPTIONAL $1"; } +has_py() { [ ${#PY[@]} -gt 0 ] && "${PY[@]}" -c "import $1" >/dev/null 2>&1; } + +echo "OS $OS ($(uname -sm))" + +if [ ${#PY[@]} -gt 0 ]; then + ok "Python 3.8+: ${PY[*]} ($("${PY[@]}" --version 2>&1))" + if has_py PIL; then ok "Python package Pillow"; else need "Python package Pillow (pip name: Pillow), for ${PY[*]}"; fi +else + need "Python 3.8+" +fi + +# OCR and faces: Apple Vision through swiftc on macOS, tesseract elsewhere. +vision=0 +if [ "$OS" = macos ] && command -v swiftc >/dev/null 2>&1; then + vision=1 + ok "Apple Vision OCR and face detection (swiftc: $(command -v swiftc))" +fi +if command -v tesseract >/dev/null 2>&1; then + langs=" $(tesseract --list-langs 2>/dev/null | tail -n +2 | tr '\n' ' ') " + have="" + for l in eng jpn; do case "$langs" in *" $l "*) have="$have $l" ;; esac; done + ok "tesseract: $(command -v tesseract) (languages:${have:- none of eng, jpn})" + case "$langs" in *" eng "*) ;; *) need "tesseract English data (eng)" ;; esac + case "$langs" in *" jpn "*) ;; *) opt "tesseract Japanese data (jpn): Japanese text is read poorly without it" ;; esac +elif [ $vision = 0 ]; then + need "an OCR engine: tesseract (or macOS with swiftc from the Xcode Command Line Tools, for Apple Vision)" +fi +if [ $vision = 0 ]; then + if has_py cv2; then + ok "Face detection: OpenCV (Python package opencv-python-headless)" + else + opt "Face detection: none (Apple Vision is macOS only; Python package opencv-python-headless adds it). Without it, faces and avatars are only found by looking at each image." + fi +fi + +# HEIC from iPhones: pillow-heif anywhere, sips on macOS, heif-convert on Linux. +if has_py pillow_heif; then + ok "HEIC conversion: pillow-heif" +elif [ "$OS" = macos ] && command -v sips >/dev/null 2>&1; then + ok "HEIC conversion: sips" +elif command -v heif-convert >/dev/null 2>&1; then + ok "HEIC conversion: heif-convert" +else + opt "HEIC conversion: none (Python package pillow-heif, or heif-convert from libheif). Only needed for .heic screenshots." +fi + +exit $missing diff --git a/.agents/skills/line-evidence/scripts/manifest.py b/.agents/skills/line-evidence/scripts/manifest.py new file mode 100755 index 00000000..1f4224ae --- /dev/null +++ b/.agents/skills/line-evidence/scripts/manifest.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Keep test//evidence/manifest.json: which screenshot proves which step. + +Usage: + manifest.py --feature test/ pick --step B5 [--platform P] [--phase before|capture|after] [--note T] RAW... + manifest.py --feature test/ unpick RAW... + manifest.py --feature test/ set --in-pr yes|no TARGET... # TARGET: step ID, raw or redacted file + manifest.py --feature test/ show + manifest.py --feature test/ eligible # files that may be uploaded + +--platform and --phase default to the step's row in PLAN.md. Shape: + + {"feature": "", "updated": "", + "steps": {"B5": [{"raw": "raw/", "platform": "LINE iOS", "phase": "after", + "redacted": "redacted/B5-line-ios-after.png" | null, + "approved": false, "approval": "" | null, + "redacted_sha256": "" | null, "in_pr": true, "note": ""}]}} + +"redacted", "approved", "approval" and "redacted_sha256" are set by +`redact.py final` only. An output stays valid only while the raw image and +its boxes still match the approval it was written under (checked while the +raw image exists; after raw/ is deleted the approved file is kept), and the +file on disk is the one that was written. Every command here (and detect.py and +redact.py) deletes outputs that no longer pass, so nothing unapproved is +left outside raw/. Upload only what `eligible` prints. +""" + +import argparse +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from evidence_lib import (PHASES, STEP, Feature, die, is_approved, load_json, now_iso, parse_plan, # noqa: E402 + save_json, sha256_file, slug) + + +def load(feat): + m = load_json(feat.manifest, None) or {"feature": feat.slug, "steps": {}} + m.setdefault("steps", {}) + return m + + +def save(feat, m): + m["feature"] = feat.slug + m["updated"] = now_iso() + m["steps"] = {k: m["steps"][k] for k in sorted(m["steps"], key=lambda s: (s[0], int(s[1:])))} + save_json(feat.manifest, m) + + +def entries(m): + for step, files in m["steps"].items(): + for e in files: + yield step, e + + +def find(m, rel_raw): + for step, e in entries(m): + if e["raw"] == rel_raw: + return step, e + return None, None + + +def current(feat, entry): + """True when the entry's redacted file is still backed by a valid approval. + + The file on disk must be the one `final` wrote. While the raw image is + there, it and its boxes must still match the approval. Once the user has + deleted raw/ (after the PR is merged), the approved file is kept as it is: + nothing can have changed what it was approved from. + """ + if not entry.get("redacted") or not entry.get("approved") or not entry.get("approval"): + return False + out = os.path.join(feat.evidence, entry["redacted"]) + if not os.path.exists(out) or sha256_file(out) != entry.get("redacted_sha256"): + return False + raw = os.path.join(feat.evidence, entry["raw"]) + if not os.path.exists(raw): + return True + data = load_json(feat.box_path(raw), None) + return is_approved(data, raw) and data["approved"]["hash"] == entry["approval"] + + +def drop_output(feat, entry): + """Delete an entry's redacted file and clear its approval.""" + if entry.get("redacted"): + out = os.path.join(feat.evidence, entry["redacted"]) + if os.path.exists(out): + os.remove(out) + print(f"removed stale {entry['redacted']}", file=sys.stderr) + entry.update(redacted=None, approved=False, approval=None, redacted_sha256=None) + + +def sweep(feat, m=None): + """Delete every redacted output that is no longer approved; save when anything changed.""" + own = m is None + m = m or load(feat) + stale = [e for _, e in entries(m) if (e.get("redacted") or e.get("approved")) and not current(feat, e)] + for e in stale: + drop_output(feat, e) + if stale and own: + save(feat, m) + return m + + +def output_name(m, step, entry): + """--.png, with -2, -3... when another entry already has the name.""" + base = f"{step}-{slug(entry['platform'])}-{entry['phase']}" + taken = {e.get("redacted") for _, e in entries(m) if e is not entry} + name, n = f"redacted/{base}.png", 1 + while name in taken: + n += 1 + name = f"redacted/{base}-{n}.png" + return name + + +def cmd_pick(feat, m, a): + if not STEP.match(a.step): + die(f"step ID {a.step!r} does not match ^[A-Z][0-9]+$") + _, steps, _ = parse_plan(feat.plan) + row = steps.get(a.step, {}) + platform = a.platform or row.get("platform") + phase = a.phase or row.get("phase") + if not platform or not phase: + die(f"{a.step} is not in {feat.plan}; pass --platform and --phase") + if phase not in PHASES: + die(f"phase {phase!r} is not one of {PHASES}") + for r in a.raw: + rel = feat.rel(feat.raw_image(r)) + old_step, old = find(m, rel) + if old: + drop_output(feat, old) + m["steps"][old_step].remove(old) + if not m["steps"][old_step]: + del m["steps"][old_step] + m["steps"].setdefault(a.step, []).append({ + "raw": rel, "platform": platform, "phase": phase, "redacted": None, + "approved": False, "in_pr": True, "note": a.note or ""}) + print(f"{a.step} {phase} {platform}: {rel}") + + +def cmd_unpick(feat, m, a): + for r in a.raw: + rel = feat.rel(feat.raw_image(r)) + step, e = find(m, rel) + if not e: + print(f"not picked: {rel}") + continue + drop_output(feat, e) + m["steps"][step].remove(e) + if not m["steps"][step]: + del m["steps"][step] + print(f"unpicked {step}: {rel}") + + +def cmd_set(feat, m, a): + flag = a.in_pr == "yes" + for t in a.targets: + hits = [e for s, e in entries(m) if t in (s, e["raw"], e.get("redacted"), os.path.basename(e["raw"]))] + if not hits: + die(f"nothing in the manifest matches {t!r}") + for e in hits: + e["in_pr"] = flag + print(f"in_pr={flag}: {e['raw']}") + + +def cmd_eligible(feat, m, a): + for step, e in entries(m): + if e.get("in_pr") and current(feat, e): + print(f"{step}\t{e['phase']}\t{e['platform']}\t{os.path.join(feat.evidence, e['redacted'])}") + + +def cmd_show(feat, m, a): + for step, e in entries(m): + state = "approved" if e.get("approved") else "not approved" + pr = "in PR" if e.get("in_pr") else "not in PR" + print(f"{step:5} {e['phase']:8} {e['platform']:20} {e.get('redacted') or '-':40} {state}, {pr} ({e['raw']})") + if e.get("redacted") and not os.path.exists(os.path.join(feat.evidence, e["redacted"])): + print(f" warning: {e['redacted']} is missing") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--feature", required=True) + ap.add_argument("--plan") + sub = ap.add_subparsers(dest="cmd", required=True) + p = sub.add_parser("pick") + p.add_argument("--step", required=True) + p.add_argument("--platform") + p.add_argument("--phase") + p.add_argument("--note") + p.add_argument("raw", nargs="+") + p = sub.add_parser("unpick") + p.add_argument("raw", nargs="+") + p = sub.add_parser("set") + p.add_argument("--in-pr", required=True, choices=["yes", "no"]) + p.add_argument("targets", nargs="+") + sub.add_parser("show") + sub.add_parser("eligible") + a = ap.parse_args() + feat = Feature(a.feature, a.plan) + m = load(feat) + sweep(feat, m) + {"pick": cmd_pick, "unpick": cmd_unpick, "set": cmd_set, "show": cmd_show, "eligible": cmd_eligible}[a.cmd](feat, m, a) + if os.path.exists(feat.manifest) or m["steps"]: + save(feat, m) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/line-evidence/scripts/redact.py b/.agents/skills/line-evidence/scripts/redact.py new file mode 100755 index 00000000..5db4e82b --- /dev/null +++ b/.agents/skills/line-evidence/scripts/redact.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Preview, approve and write redacted screenshots from evidence/boxes/.json. + +Usage: + redact.py --feature test/ preview IMAGE... # raw/previews/.png + raw/review.html + redact.py --feature test/ approve IMAGE... # record the user's OK for the current boxes + redact.py --feature test/ final IMAGE... # redacted/--.png + +Boxes that get covered: every "auto" box whose id is not in "unhide", every +"kept" box whose id is in "hide", and every "add" rectangle +({"x", "y", "w", "h", "label"} in pixels). Covers are solid filled boxes, +never blur, padded a little so letter edges do not show. + +The preview puts the original with numbered outlines (red: covered, green: +left visible, blue: added by hand) next to the redacted result, and prints +the number -> id table. Previews contain raw pixels and stay in raw/. + +`approve` stores a hash of the raw image's bytes and the covered +rectangles; replacing the screenshot or editing any box voids it, and every +script then deletes the redacted output made under the old approval. +`final` refuses images without a matching approval or without a manifest +entry (manifest.py pick), re-encodes to PNG +with every piece of metadata dropped (EXIF, GPS, text chunks, ICC after +converting to sRGB), OCRs the result again as a check, and records it in +manifest.json. +""" + +import argparse +import html +import io +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from evidence_lib import (Feature, approval_hash, covers, die, is_approved, load_json, now_iso, # noqa: E402 + numbered, parse_plan, rect, save_json, sha256_file) + +try: + from PIL import Image, ImageDraw, ImageFont +except ImportError: + die("Pillow is missing; run the preflight") + +FILL = (24, 24, 24) +COLORS = {"cover": (230, 40, 40), "keep": (30, 170, 60), "add": (40, 110, 240)} + + +def load_boxes(feat, image): + path = feat.box_path(image) + data = load_json(path, None) + if data is None: + die(f"no boxes for {os.path.basename(image)}; run detect.py first") + return path, data + + +def render(im, data): + out = im.convert("RGB") + draw = ImageDraw.Draw(out) + for b in covers(data): + draw.rectangle(rect(b, out.size), fill=FILL) + return out + + +def font(size): + for name in ("Arial Bold.ttf", "Arial.ttf", "DejaVuSans-Bold.ttf", "arialbd.ttf", "arial.ttf"): + try: + return ImageFont.truetype(name, size) + except OSError: + pass + return ImageFont.load_default() + + +def preview(feat, image): + _, data = load_boxes(feat, image) + with Image.open(image) as im: + im.load() + marked = im.convert("RGB") + draw = ImageDraw.Draw(marked) + f = font(max(12, im.size[0] // 70)) + boxes = numbered(data) + for b in boxes: + r = rect(b, im.size) + draw.rectangle(r, outline=COLORS[b["state"]], width=max(2, im.size[0] // 500)) + draw.text((r[0], max(0, r[1] - f.size - 2)), str(b["n"]), fill=COLORS[b["state"]], font=f) + after = render(im, data) + gap = 24 + sheet = Image.new("RGB", (im.size[0] * 2 + gap, im.size[1]), (255, 255, 255)) + sheet.paste(marked, (0, 0)) + sheet.paste(after, (im.size[0] + gap, 0)) + os.makedirs(feat.previews, mode=0o700, exist_ok=True) + out = feat.preview_path(image) + sheet.save(out) + print(f"{os.path.basename(image)} -> {out}") + for b in boxes: + what = b.get("text") or b.get("label") or (f"{b.get('chars')} chars" if b.get("chars") else "") + print(f" {b['n']:3} {b['state']:5} {b['kind']:9} {b['id']:9} {b['x']},{b['y']} {b['w']}x{b['h']} {what}") + if not data.get("faces_checked"): + print(" faces NOT checked by a detector: look for faces and avatars yourself") + + +def write_review(feat, items): + rows = [] + for image, out, boxes, data in items: + approved = is_approved(data, image) + table = "".join( + f"{b['n']}{b['state']}{html.escape(b['kind'])}" + f"{b['id']}{html.escape(b.get('text') or b.get('label') or '')}" + for b in boxes) + rows.append( + f"

{html.escape(os.path.basename(image))} " + f"{'approved' if approved else 'not approved'}

" + f"" + f"
{len(boxes)} boxes{table}
") + page = f"""Redaction review + +

Local file with unredacted screenshots. Do not upload, publish or share it. +Left: original with numbered boxes (red covered, green visible, blue added). Right: what will be published.

+{''.join(rows)}""" + path = os.path.join(feat.raw, "review.html") + with open(path, "w", encoding="utf-8") as fh: + fh.write(page) + print(f"review page: {path}") + + +def to_srgb(im): + icc = im.info.get("icc_profile") + if not icc: + return im + try: + from PIL import ImageCms + + src = ImageCms.ImageCmsProfile(io.BytesIO(icc)) + mode = "RGBA" if im.mode in ("RGBA", "LA", "P") else "RGB" + return ImageCms.profileToProfile(im.convert(mode), src, ImageCms.createProfile("sRGB"), outputMode=mode) + except Exception as e: + print(f" note: could not convert ICC profile to sRGB ({e}); dropping it", file=sys.stderr) + return im + + +def final(feat, image, manifest_mod): + path, data = load_boxes(feat, image) + size = (data["width"], data["height"]) + # Drop outputs whose approval no longer holds before anything else. + m = manifest_mod.sweep(feat) + if not is_approved(data, image): + die(f"{os.path.basename(image)} is not approved for its current image and boxes; " + "preview it and get the user's OK") + step, entry = manifest_mod.find(m, feat.rel(image)) + if not entry: + die(f"{os.path.basename(image)} is not in the manifest; run manifest.py pick first") + with Image.open(image) as im: + im.load() + if im.size != size: + die(f"{os.path.basename(image)} is {im.size}, boxes were made for {size}; run detect.py again") + im = to_srgb(im) + alpha = "A" in im.getbands() or (im.mode == "P" and "transparency" in im.info) + mode = "RGBA" if alpha else "RGB" + base = im.convert(mode) + drawn = base.copy() + draw = ImageDraw.Draw(drawn) + for b in covers(data): + draw.rectangle(rect(b, size), fill=FILL + ((255,) if mode == "RGBA" else ())) + # Rebuild from raw pixels so no info, EXIF, ICC or text chunk can survive. + clean = Image.frombytes(mode, size, drawn.tobytes()) + if entry.get("redacted"): + manifest_mod.drop_output(feat, entry) + rel = manifest_mod.output_name(m, step, entry) + out = os.path.join(feat.evidence, rel) + os.makedirs(feat.redacted, exist_ok=True) + clean.save(out, "PNG", optimize=True) + with Image.open(out) as check: + leftover = set(check.info) - {"dpi", "gamma", "transparency"} + if leftover or check.getexif(): + os.remove(out) + die(f"metadata survived in {out}: {sorted(leftover)}") + verify(feat, out, data) + entry.update(redacted=rel, approved=True, approval=data["approved"]["hash"], redacted_sha256=sha256_file(out)) + manifest_mod.save(feat, m) + print(f"{os.path.basename(image)} -> {out}") + + +def verify(feat, out, data): + """OCR the redacted file and warn about text that would have been hidden.""" + import detect + + test_data, _, _ = parse_plan(feat.plan) + try: + _, result = detect.analyze(out, "auto") + except SystemExit: + print(" check skipped: no OCR engine", file=sys.stderr) + return + auto, _ = detect.classify(result, test_data, data.get("allow", [])) + unhidden = set(data.get("unhide", [])) + stray = [b for b in auto if b["id"] not in unhidden and b["kind"] != "face"] + stray = [b for b in stray if not any(_inside(b, u) for u in data["auto"] if u["id"] in unhidden)] + faces = [b for b in auto if b["kind"] == "face"] + if stray or faces: + print(f" CHECK: OCR still finds {len(stray)} hidden-class text box(es) and {len(faces)} face(s) in the output:") + for b in stray + faces: + print(f" {b['kind']} at {b['x']},{b['y']} {b['w']}x{b['h']}") + print(" Look at the output before using it; add boxes and redo if anything is readable.") + + +def _inside(a, b): + return a["x"] >= b["x"] - 4 and a["y"] >= b["y"] - 4 and a["x"] + a["w"] <= b["x"] + b["w"] + 4 \ + and a["y"] + a["h"] <= b["y"] + b["h"] + 4 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--feature", required=True) + ap.add_argument("--plan") + ap.add_argument("cmd", choices=["preview", "approve", "final"]) + ap.add_argument("images", nargs="+") + a = ap.parse_args() + feat = Feature(a.feature, a.plan) + images = [feat.raw_image(i) for i in a.images] + if a.cmd == "preview": + for image in images: + preview(feat, image) + # The review page lists every image that has a preview, not just this run's. + items = [] + for e in feat.load_index() or [{"file": f} for f in sorted(os.listdir(feat.raw))]: + image = os.path.join(feat.raw, e["file"]) + out = feat.preview_path(image) + if os.path.exists(out) and os.path.exists(feat.box_path(image)): + _, data = load_boxes(feat, image) + items.append((image, out, numbered(data), data)) + write_review(feat, items) + elif a.cmd == "approve": + for image in images: + path, data = load_boxes(feat, image) + data["approved"] = {"hash": approval_hash(data, image), "at": now_iso()} + save_json(path, data) + print(f"approved {os.path.basename(image)} ({len(covers(data))} covers)") + # A new approval makes any output written under the old one stale. + import manifest + + manifest.sweep(feat) + else: + import manifest + + for image in images: + final(feat, image, manifest) + + +if __name__ == "__main__": + main() diff --git a/.agents/skills/line-evidence/scripts/vision-detect.swift b/.agents/skills/line-evidence/scripts/vision-detect.swift new file mode 100644 index 00000000..3971c816 --- /dev/null +++ b/.agents/skills/line-evidence/scripts/vision-detect.swift @@ -0,0 +1,68 @@ +// On-device text and face detection with Apple Vision (macOS only). +// +// Usage: swift vision-detect.swift (detect.py compiles and caches it) +// +// Prints JSON to stdout: {"width", "height", "words": [...], "faces": [...]}. +// Every box is {"x", "y", "w", "h"} in pixels with the origin at the top left. +// Words carry "text", "line" (index of the OCR line) and "conf". Nothing is +// sent anywhere: Vision runs on the device. + +import AppKit +import Foundation +import Vision + +func fail(_ msg: String) -> Never { + FileHandle.standardError.write((msg + "\n").data(using: .utf8)!) + exit(1) +} + +guard CommandLine.arguments.count == 2 else { fail("usage: vision-detect ") } +let url = URL(fileURLWithPath: CommandLine.arguments[1]) +guard let image = NSImage(contentsOf: url), + let cg = image.cgImage(forProposedRect: nil, context: nil, hints: nil) +else { fail("cannot read image: \(url.path)") } +let W = Double(cg.width), H = Double(cg.height) + +func box(_ r: CGRect) -> [String: Double] { + // Vision uses normalized coordinates with the origin at the bottom left. + return ["x": (r.minX * W).rounded(), "y": ((1 - r.maxY) * H).rounded(), + "w": (r.width * W).rounded(), "h": (r.height * H).rounded()] +} + +let text = VNRecognizeTextRequest() +text.recognitionLevel = .accurate +text.usesLanguageCorrection = false +text.automaticallyDetectsLanguage = true +let faces = VNDetectFaceRectanglesRequest() + +do { + try VNImageRequestHandler(cgImage: cg, options: [:]).perform([text, faces]) +} catch { fail("vision failed: \(error)") } + +var words: [[String: Any]] = [] +for (lineIndex, obs) in (text.results ?? []).enumerated() { + guard let cand = obs.topCandidates(1).first else { continue } + let s = cand.string + // Split the line on whitespace and ask Vision for each word's own box, so a + // single name can be hidden without covering the rest of the line. + var idx = s.startIndex + while idx < s.endIndex { + while idx < s.endIndex && s[idx].isWhitespace { idx = s.index(after: idx) } + var end = idx + while end < s.endIndex && !s[end].isWhitespace { end = s.index(after: end) } + if idx < end { + let r = (try? cand.boundingBox(for: idx..