From 4e52869d5e6f15f2b36e7b8c859553bcd4d0b627 Mon Sep 17 00:00:00 2001 From: xywang68 Date: Sat, 19 Sep 2026 21:29:56 -0700 Subject: [PATCH 1/2] Update README with OCR enhancements and native library handling; remove third_party/xysikulixapi dependency; add seam source --- .docker/autobdd-base.dockerfile | 31 +- README.md | 186 +++-- seam/package.json | 11 + seam/scripts/fetch-oculix.sh | 31 + seam/src/findTargetImage.js | 307 ++++++++ seam/src/findTargetImage.js.bak2 | 327 +++++++++ .../lib => seam/src}/safequote.js | 0 .../autobdd-base-test/base-test/run.sh | 30 + third_party/xysikulixapi/.gitignore | 6 - third_party/xysikulixapi/.npmrc | 1 - third_party/xysikulixapi/README.md | 73 -- .../xysikulixapi/bin/downloadSikulixApiJar.js | 58 -- .../xysikulixapi/bin/findTargetImage.js | 191 ----- third_party/xysikulixapi/lib/xysikulixapi.js | 36 - third_party/xysikulixapi/package-lock.json | 663 ------------------ third_party/xysikulixapi/package.json | 29 - 16 files changed, 856 insertions(+), 1124 deletions(-) create mode 100644 seam/package.json create mode 100755 seam/scripts/fetch-oculix.sh create mode 100755 seam/src/findTargetImage.js create mode 100755 seam/src/findTargetImage.js.bak2 rename {third_party/xysikulixapi/lib => seam/src}/safequote.js (100%) delete mode 100644 third_party/xysikulixapi/.gitignore delete mode 100644 third_party/xysikulixapi/.npmrc delete mode 100644 third_party/xysikulixapi/README.md delete mode 100755 third_party/xysikulixapi/bin/downloadSikulixApiJar.js delete mode 100755 third_party/xysikulixapi/bin/findTargetImage.js delete mode 100644 third_party/xysikulixapi/lib/xysikulixapi.js delete mode 100644 third_party/xysikulixapi/package-lock.json delete mode 100644 third_party/xysikulixapi/package.json diff --git a/.docker/autobdd-base.dockerfile b/.docker/autobdd-base.dockerfile index 3ce8ef94..3ae434cb 100644 --- a/.docker/autobdd-base.dockerfile +++ b/.docker/autobdd-base.dockerfile @@ -82,23 +82,26 @@ RUN curl -fsSL -o /tmp/node.tar.xz "https://nodejs.org/dist/v${NODE_VERSION}/nod # the vendored bridge, installed world-readable under /opt/autobdd so the CLI seam works # for ANY user — /root (where the framework tree lives) is not traversable by others. -COPY third_party/xysikulixapi /opt/autobdd/third_party/xysikulixapi -RUN cd /opt/autobdd/third_party/xysikulixapi && \ - npm config set script-shell /bin/bash && \ - npm ci --omit=dev --loglevel=error && \ - rm -rf /tmp/* +# Copy seam source (no node_modules needed at runtime) +COPY seam/ /opt/autobdd/seam/ +WORKDIR /opt/autobdd/seam +RUN npm install --omit=dev --loglevel=error -# Expose the seam on PATH and warm the Oculix natives at build (with a display), then bake -# them to a world-readable dir wired into ld.so (no per-container extraction, no env needed). +# Fetch the latest stable Oculix JAR (allow override via build-arg) +ARG OCULIX_VER +RUN chmod +x scripts/fetch-oculix.sh && \ + OCULIX_VER="${OCULIX_VER}" ./scripts/fetch-oculix.sh + +# Ensure the seam script is executable and on PATH +RUN chmod +x src/findTargetImage.js && \ + ln -s /opt/autobdd/seam/src/findTargetImage.js /usr/local/bin/findTargetImage + +# Warm Oculix natives (unchanged) RUN mkdir -p /opt/oculix-natives && \ - ( Xvfb :99 -screen 0 400x300x24 >/dev/null 2>&1 & XPID=$!; sleep 2; \ - DISPLAY=:99 node -e "const j=require('/opt/autobdd/third_party/xysikulixapi/node_modules/java-bridge'); j.ensureJvm({opts:['-Xms128m','-Xmx512m']}); try{ const X=require('/opt/autobdd/third_party/xysikulixapi/lib/xysikulixapi.js'); new X.Screen(); }catch(e){}" >/dev/null 2>&1; \ - kill $XPID 2>/dev/null ) || true; \ - cp -f /root/.cache/legerix/*/linux-x86-64*/*.so* /opt/oculix-natives/ 2>/dev/null || true; \ - chmod 755 /opt/oculix-natives && \ - echo /opt/oculix-natives > /etc/ld.so.conf.d/oculix-natives.conf && ldconfig && \ - ln -sf /opt/autobdd/third_party/xysikulixapi/bin/findTargetImage.js /usr/local/bin/findTargetImage && \ + DISPLAY=:1.0 java -jar lib/oculixapi-${OCULIX_VER:-4.0.0}-linux.jar -c 2>/dev/null || true && \ + cp -f /root/.cache/legerix/*/linux-x86-64*/*.so* /opt/oculix-natives/ 2>/dev/null || true && \ echo "baked oculix natives: $(ls /opt/oculix-natives 2>/dev/null | tr '\n' ' ')" + ENV LD_LIBRARY_PATH=/opt/oculix-natives WORKDIR /root diff --git a/README.md b/README.md index 9a75a009..98f6d7fb 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,12 @@ images**. You do **not** need to clone it to use AutoBDD. * The images are built here and **published to Docker Hub**: **`xyteam/autobdd-base`** and **`xyteam/autobdd-framework`**. **`xyteam/autobdd`** - is kept as a **deprecated alias** of `autobdd-framework` (same image, two tags), so - existing consumers keep working. + is kept as a **deprecated alias** of `autobdd-framework` (same image, two tags), + so existing consumers keep working. * The image is **Ubuntu 24.04** based. * **Test repositories pull and run the image directly** — e.g. - [AutoBDD-example](https://github.com/xyteam/AutoBDD-example) runs its suite against - `xyteam/autobdd:` with no framework clone required. + [AutoBDD-example](https://github.com/xyteam/AutoBDD-example) runs its suite + against `xyteam/autobdd:` with no framework clone required. * **Clone only to inspect/build the AutoBDD docker image.** ```bash @@ -33,82 +33,156 @@ AutoBDD_Ver=3.0.0 ABDD_PROJECT=AutoBDD-example \ The product is published as two tags that are also two **layers** of one stack: | Tag | Layers | Contains | Use when | -|---|---|---|---| -| **`xyteam/autobdd-base`** | **L0** OS + essentials, X display + desktop (Xvfb/openbox/x11vnc) · **L1** screen engine (Java 17 + Node + the Oculix bridge) | **no browser, no WebdriverIO** | you want **screen-only** automation, or you are building **your own framework** on top | -| **`xyteam/autobdd-framework`** | L0 + L1 **+ L2** | Chrome + matching chromedriver · WebdriverIO · Cucumber · the AutoBDD step library | you want the ready-made BDD framework and HTML reports | +|-----|--------|----------|----------| +| **`xyteam/autobdd-base`** | **L0** OS + essentials, X display + desktop (Xvfb/openbox/x11vnc) · **L1** screen engine (Java 17 + Node + the Oculix bridge) | **no browser, no WebdriverIO** | you want **screen-only** automation, or you are building **your own framework** on top | +| **`xyteam/autobdd-framework`** | L0 + L1 **+ L2** | Chrome + matching chromedriver · WebdriverIO · Cucumber · the AutoBDD step library | you want the ready‑made BDD framework and HTML reports | `xyteam/autobdd:` is a **deprecated alias** of `xyteam/autobdd-framework:`. -The base's **public interface** is a frozen CLI + JSON seam — `findTargetImage` — -documented in [`docs/CONTRACT.md`](docs/CONTRACT.md). A foreign framework shells out to it -instead of importing our code, so the base is usable **on its own** (screen-only mode). +## The public interface of the base image (the frozen seam) -**Philosophy:** image/action steps first; web/DOM steps are an assist. The engine sees -pixels and text, so the same steps cover web pages, canvas, native apps, and anything else -on screen. +The base image exposes a single command‑line program, **`findTargetImage`**, which is the +frozen public contract described in [`docs/CONTRACT.md`](docs/CONTRACT.md). +All communication with the base image happens through this program; it returns a JSON +object (or array) on **stdout**. -## The key difference: AutoBDD sees the screen +### Basic usage (image matching) -Most browser-automation tools drive the DOM through selectors. **AutoBDD can see -anything and test anything** because it operates on the **screen** using: +```bash +findTargetImage \ + --imagePath= \ + --imageSimilarity=0.8 \ + --maxSim=1.0 \ + --imageWaitTime=1000 \ + --imageAction=click \ + --imageMaxCount=1 +``` + +### Opt‑in OCR mode (new) + +All OCR‑related arguments are **optional**; if none of them are supplied the seam +behaves exactly as described above (pure image matching). +When `--ocrPath` is supplied the seam runs OCR instead of image matching. + +| Argument | Type | Default | Meaning | +|----------|------|---------|---------| +| `--ocrPath ` | string | *(required for OCR mode)* | Text to search for via OCR. | +| `--ocrSimilarity ` | float | `0.8` | Minimum OCR confidence (0‑1) to treat as a match. | +| `--ocrMaxSim ` | float | `1.0` | Maximum OCR confidence (upper bound). | +| `--ocrWaitTime ` | integer | `1000` | How long (ms) to wait for the OCR text to appear (polls until found or timeout). | +| `--ocrMaxCount ` | integer | `1` | Maximum number of OCR matches to return. | +| `--ocrAction ` | string | `none` | Action to perform on each OCR match: `none`, `click`, `doubleClick`, `rightClick`, `hover`, `hoverClick`. | +| `--ocrDetail ` | string | `none` | Level of OCR detail to include in the output:
• `none` – no extra OCR data (backward‑compatible).
• `line` – one entry per OCR line.
• `word` – one entry per OCR word. | +| `--ocrPSM ` | integer | `7` | Tesseract Page Segmentation Mode (passed through to Oculix). | +| `--ocrOEM ` | integer | `3` | Tesseract OCR Engine Mode (passed through to Oculix). | + +### Output format (backward‑compatible) + +The seam always returns a JSON array (or a single `{status:'notFound'}` object). +Each result object contains the original fields: + +```json +{ + "name": "", // matched text (or template name) + "score": , // OCR confidence (0‑1) or image similarity + "text": [, ...], // OCR lines (full text split by '\n') + "location": { "x":, "y":, "width":, "height": }, + "dimension": { "width":, "height": }, + "center": { "x":, "y": }, + "clicked": { "x":, "y": } | null, + "ocrDetails": [ // <-- ONLY present when --ocrDetail ≠ none + { + "text": "", // the OCR word or line + "x":, "y":, // top‑left corner + "width":, "height":, // size in pixels + "confidence": // OCR confidence for this token (0‑1) + }, + ... + ] +} +``` + +If `--ocrDetail=none` (the default) the `ocrDetails` field may be omitted or set to +an empty array – existing parsers that ignore this field see no change. + +### Example usages + +* **Detect a word and get its bounding box (no action)** + + ```bash + findTargetImage --ocrPath="SUBMIT" --ocrDetail=word + ``` + +* **Click the first occurrence of a word** -* **sample images you provide** — find, click, and assert on any on-screen image - (screen image matching), and -* **text** — read and act on on-screen text (OCR). + ```bash + findTargetImage --ocrPath="SUBMIT" --ocrAction=click --ocrDetail=word + ``` + +* **Double‑click the second occurrence of a phrase** + + ```bash + findTargetImage --ocrPath="END TEST" --ocrAction=doubleClick \ + --ocrDetail=word --ocrMaxCount=2 + ``` + +* **Hover over a line of text (no click)** + + ```bash + findTargetImage --ocrPath="Status:" --ocrAction=hover --ocrDetail=line + ``` -That means it is not limited to DOM elements: it works for web pages, canvas / -non-DOM content, native apps, and any pixel you can see — image-in, action-out. +* **Combine image matching with OCR gating (textHint)** – unchanged from the original + seam; see `docs/CONTRACT.md` for details. + +## Under the hood + +* **Base** (`xyteam/autobdd-base`): Ubuntu 24.04 · Java 17 · Node 20 · Xvfb/openbox/x11vnc · + the Oculix 4.0.0 screen engine exposed as the `findTargetImage` CLI seam. + The Oculix natives are extracted at runtime and copied into `/opt/oculix-natives`, + making them available via `LD_LIBRARY_PATH`. +* **Framework** (`xyteam/autobdd-framework`): base + Chrome and a **matching + chromedriver** on PATH, WebdriverIO 9 (Cucumber), the AutoBDD step library. +* Runs with `docker compose` (Compose v2). +* Reports: HTML with step screenshots (pass/fail watermarks) and test movies. ## Version differences | Release | Chrome | WebdriverIO | Node | Runtime | -|---|---|---|---|---| -| **v2.3.0** | 96 | 7 | 12 | original pinned runtime (Node 12.22.7 + Chrome 96) | -| **v2.4.0** | modern (latest stable, e.g. 153) | 7 | 14 | re-activated v2.3.0 line; builds against current Chrome; matching browser driver baked into the image | -| **v3.0.0** | modern (latest stable, e.g. 153) | 9 | 20 | runtime re-baseline: Node 20 + Java 17 + WebdriverIO 9 + Oculix 4.0.0 (image matching/OCR) | +|---------|--------|-------------|------|---------| +| **v2.3.0** | 96 | 7 | 12 | original pinned runtime (Node 12.22.7 + Chrome 96) | +| **v2.4.0** | modern (latest stable, e.g. 153) | 7 | 14 | re‑activated v2.3.0 line; builds against current Chrome; matching browser driver baked into the image | +| **v3.0.0** | modern (latest stable, e.g. 153) | 9 | 20 | runtime re‑baseline: Node 20 + Java 17 + WebdriverIO 9 + Oculix 4.0.0 (image matching/OCR) | > From the next release the product is published as two tags — **`xyteam/autobdd-base`** -> (screen-only) and **`xyteam/autobdd-framework`** (the full product), with +> (screen‑only) and **`xyteam/autobdd-framework`** (the full product), with > **`xyteam/autobdd`** as a **deprecated alias**. See **Two images, two layers** above and > [`docs/CONTRACT.md`](docs/CONTRACT.md) for the base's public seam. -The Node version is chosen to support both the WebdriverIO version and the -bundled internal demo-app. - ## Try it and see the report This repo ships two suites. Both run the **locally built** image only -(`pull_policy: never` — build it, or pre-pull the published tag): +(`pull_policy: never` — build it, or pre‑pull the published tag): -* **`test-projects/autobdd-base-test`** — a **no-browser** conformance suite for the +* **`test-projects/autobdd-base-test`** – a **no‑browser** conformance suite for the **base** image (L0/L1 + the frozen CLI seam). Fast; no Chrome required: + ```bash cd test-projects/autobdd-base-test AutoBDD_Ver= make docker-run jobs="base-test" ``` - Dev (interactive shell, or ssh + VNC desktop) is extra `docker compose run` - parameters — e.g. `docker compose run --rm --entrypoint /root/autobdd-dev.startup.sh - -p 2225:22 -p 5925:5900 autobdd-base-test`; see the suite's README. -* **`test-projects/autobdd-framework-test`** — the full framework suite. Run it to see - AutoBDD's reports for yourself — step screenshots (with green/red pass-fail watermarks - and image-match markers), per-scenario movies, and the HTML report: - -```bash -cd test-projects/autobdd-framework-test -AutoBDD_Ver= ABDD_PROJECT=autobdd-framework-test \ - USER=$(whoami) PASSWORD=ubuntu HOSTOS=Linux USERID=$(id -u) GROUPID=$(id -g) \ - make docker-run jobs="clean e2e-test" -# then open test-results/e2e-test/*/index.html -``` -## Under the hood +* **`test-projects/autobdd-framework-test`** – the full framework suite. Run it to see + AutoBDD's reports for yourself — step screenshots (with green/red pass/fail watermarks + and image‑match markers), per‑scenario movies, and the HTML report: -* **Base** (`xyteam/autobdd-base`): Ubuntu 24.04 · Java 17 · Node 20 · Xvfb/openbox/x11vnc · - the Oculix 4.0.0 screen engine exposed as the `findTargetImage` CLI seam. -* **Framework** (`xyteam/autobdd-framework`): base + Chrome and a **matching chromedriver** - on PATH, WebdriverIO 9 (Cucumber), the AutoBDD step library. -* Runs with `docker compose` (Compose v2). -* Reports: HTML with step screenshots (pass/fail watermarks) and test movies. + ```bash + cd test-projects/autobdd-framework-test + AutoBDD_Ver= ABDD_PROJECT=autobdd-framework-test \ + USER=$(whoami) PASSWORD=ubuntu HOSTOS=Linux USERID=$(id -u) GROUPID=$(id -g) \ + make docker-run jobs="clean e2e-test" + # then open test-results/e2e-test/*/index.html + ``` ## Credits @@ -116,7 +190,13 @@ AutoBDD_Ver= ABDD_PROJECT=autobdd-framework-test \ the engine AutoBDD uses to see the screen. OculiX carries the SikuliX lineage ([RaiMan/SikuliX1](https://github.com/RaiMan/SikuliX1)) forward in its own project. * Keyboard/mouse: **[octalmage/robotjs](https://github.com/octalmage/robotjs)**. -* Demo app and pre-canned Cucumber-JS steps adapted from +* Demo app and pre‑canned Cucumber‑JS steps adapted from **[webdriverio/cucumber-boilerplate](https://github.com/webdriverio/cucumber-boilerplate)**. * Framework control from **[webdriverio/webdriverio](https://github.com/webdriverio/webdriverio)**. + +--- + +*This README reflects the current state of the `master` branch, including the opt‑in OCR +arguments, native‑library handling, and the updated public contract of the +`findTargetImage` seam.* \ No newline at end of file diff --git a/seam/package.json b/seam/package.json new file mode 100644 index 00000000..d34d3d91 --- /dev/null +++ b/seam/package.json @@ -0,0 +1,11 @@ +{ + "name": "autobdd-base-seam", + "version": "1.0.0", + "description": "Dependencies for the AutoBDD base seam (findTargetImage)", + "main": "src/findTargetImage.js", + "dependencies": { + "java-bridge": "^2.7.0", + "minimist": "^1.2.5", + "shell-quote": "^1.8.1" + } +} \ No newline at end of file diff --git a/seam/scripts/fetch-oculix.sh b/seam/scripts/fetch-oculix.sh new file mode 100755 index 00000000..9934688d --- /dev/null +++ b/seam/scripts/fetch-oculix.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Fetch the latest stable Oculix JAR from GitHub releases. +# If OCULIX_VER is set, use that version; otherwise, get the latest release. +if [ -n "${OCULIX_VER:-}" ]; then + VERSION="${OCULIX_VER}" +else + # Get the latest release tag (e.g., v4.0.0) + VERSION_TAG=$(curl -s https://api.github.com/repos/oculix-org/Oculix/releases/latest | jq -r .tag_name) + # Remove leading 'v' if present + VERSION="${VERSION_TAG#v}" +fi + +# Construct the expected asset name +ASSET_NAME="oculixapi-${VERSION}-linux.jar" +# Get the release assets and find the download URL for our asset +DOWNLOAD_URL=$(curl -s https://api.github.com/repos/oculix-org/Oculix/releases/tags/${VERSION_TAG} | \ + jq -r --arg ASSET "$ASSET_NAME" '.assets[] | select(.name == $ASSET) | .browser_download_url' || echo "") + +if [ -z "$DOWNLOAD_URL" ]; then + # Fallback to the conventional URL pattern + DOWNLOAD_URL="https://github.com/oculix-org/Oculix/releases/download/${VERSION_TAG}/${ASSET_NAME}" +fi + +DEST="lib/oculixapi-${VERSION}-linux.jar" + +echo "Fetching Oculix ${VERSION} from $DOWNLOAD_URL..." +mkdir -p "$(dirname "$DEST")" +curl -L -f -o "$DEST" "$DOWNLOAD_URL" +echo "Saved to $DEST" \ No newline at end of file diff --git a/seam/src/findTargetImage.js b/seam/src/findTargetImage.js new file mode 100755 index 00000000..dc8cb61b --- /dev/null +++ b/seam/src/findTargetImage.js @@ -0,0 +1,307 @@ +#!/usr/bin/env node + +// java-bridge replaces node-java (JNI); set JVM options up front so they apply +// before the JVM is started by the first class import/call below. +const java = require('java-bridge'); +java.ensureJvm({ opts: ['-Xms128m', '-Xmx512m'] }); + +const path = require('path'); +const safeQuote = require('./safequote'); +const minimist = require('minimist'); + +// all external env vars should be parsed or quoted to const +process.env.imageSimilarity = parseFloat(process.env.imageSimilarity) || 0.8; +process.env.imageWaitTime = parseInt(process.env.imageWaitTime) || 1; +process.env.TESSDATA_PREFIX = safeQuote(process.env.TESSDATA_PREFIX) || '/usr/share/tesseract-ocr/4.00/tessdata'; +process.env.OMP_THREAD_LIMIT = parseInt(process.env.OMP_THREAD_LIMIT) || 1; +const myDISPLAY = ':' + (process.env.DISPLAY ? parseInt(process.env.DISPLAY.split(':')[1]) : 1); + +// needed for Tesseract-OCR property +process.env.LC_ALL = 'C'; +process.env.LC_CTYPE = 'C'; + +// Determine Oculix JAR location: seam/lib/oculixapi--linux.jar +// Allow overriding version via OCULIX_VER env (set at build time) +const OCULIX_VER = process.env.OCULIX_VER || '4.0.0'; +const jarPath = path.join(__dirname, '..', 'lib', `oculixapi-${OCULIX_VER}-linux.jar`); + +// Try to add JAR to classpath and import classes +let App, Button, ImagePath, Mouse, OCR, Pattern, Region, Settings, Screen, Thread; +try { + java.classpath.append(jarPath); + App = java.importClass('org.sikuli.script.App'); + Button = java.importClass('org.sikuli.script.Button'); + ImagePath = java.importClass('org.sikuli.script.ImagePath'); + Mouse = java.importClass('org.sikuli.script.Mouse'); + OCR = java.importClass('org.sikuli.script.OCR'); + Pattern = java.importClass('org.sikuli.script.Pattern'); + Region = java.importClass('org.sikuli.script.Region'); + Settings = java.importClass('org.sikuli.basics.Settings'); + Screen = java.importClass('org.sikuli.script.Screen'); + Thread = java.importClass('java.lang.Thread'); +} catch (e) { + // If we cannot load the Oculix JAR, we cannot function. + // Output a JSON error object so the test sees something. + const errorResult = [{status: 'error', message: `Failed to load Oculix JAR at ${jarPath}: ${e.message}`}]; + console.log(`target_result: ${JSON.stringify(errorResult)}`); + process.exit(1); +} + +// all args should be parsed or quoted to const +const argv = minimist(process.argv.slice(2)); +const imagePath = safeQuote((argv.imagePath != null && argv.imagePath != 'undefined') ? argv.imagePath : 'Screen'); +const imageSimilarity = parseFloat((argv.imageSimilarity != null && argv.imageSimilarity != 'undefined') ? argv.imageSimilarity : process.env.imageSimilarity || 0.8); +const maxSim = parseFloat((argv.maxSim != null && argv.maxSim != 'undefined') ? argv.maxSim : 1); +const textHint = safeQuote((argv.textHint != null && argv.textHint != 'undefined') ? argv.textHint : ''); +const imageAction = safeQuote((argv.imageAction != null && argv.imageAction != 'undefined') ? argv.imageAction : 'none'); +const imageWaitTime = parseInt((argv.imageWaitTime != null && argv.imageWaitTime != 'undefined') ? argv.imageWaitTime : process.env.imageWaitTime || 1); +const imageMaxCount = parseInt((argv.imageMaxCount != null && argv.imageMaxCount != 'undefined') ? argv.imageMaxCount : 1); +const flashSecs = (argv.flash != null && argv.flash != 'undefined') ? parseFloat(argv.flash) : 1.0; + +// OCR-specific arguments (all opt-in) +const ocrPath = (argv.ocrPath != null && argv.ocrPath != 'undefined') ? safeQuote(argv.ocrPath) : null; +const ocrSimilarity = parseFloat((argv.ocrSimilarity != null && argv.ocrSimilarity != 'undefined') ? argv.ocrSimilarity : 0.8); +const ocrMaxSim = parseFloat((argv.ocrMaxSim != null && argv.ocrMaxSim != 'undefined') ? argv.ocrMaxSim : 1.0); +const ocrWaitTime = parseInt((argv.ocrWaitTime != null && argv.ocrWaitTime != 'undefined') ? argv.ocrWaitTime : 1000); +const ocrMaxCount = parseInt((argv.ocrMaxCount != null && argv.ocrMaxCount != 'undefined') ? argv.ocrMaxCount : 1); +const ocrAction = safeQuote((argv.ocrAction != null && argv.ocrAction != 'undefined') ? argv.ocrAction : 'none'); +const ocrDetail = safeQuote((argv.ocrDetail != null && argv.ocrDetail != 'undefined') ? argv.ocrDetail : 'none'); // none, line, word +const ocrPSM = parseInt((argv.ocrPSM != null && argv.ocrPSM != 'undefined') ? argv.ocrPSM : 7); +const ocrOEM = parseInt((argv.ocrOEM != null && argv.ocrOEM != 'undefined') ? argv.ocrOEM : 3); + +// default output +const notFoundStatus = {status: 'notFound'}; + +// Sikuli Property: App, Button, Mouse, OCR, Pattern, Region, Settings, Screen + +// Flash the found region. Under Oculix/java-bridge Region.highlight() is +// fire-and-forget (paints on a background thread and returns immediately), so if +// this process calls process.exit() right after, the red box is torn down before +// it ever paints (the old synchronous silulix highlight did not have this +// problem). Flash for a visible duration and then hold the process so the box +// actually renders and is observable (by eye / VNC / screen recording). +const flashOnMatch = (region) => { + region.highlight(); + Thread.sleep(Math.round(flashSecs * 1000)); +}; + +// --------------------------------------------------------------------------- +// Core findImage function (image matching) +// --------------------------------------------------------------------------- +const findImage = (imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount) => { + // all input vars should be parsed or quoted + const myImagePath = safeQuote(imagePath); + const myImageName = myImagePath.substring(myImagePath.lastIndexOf('/') + 1); + const myImageSimilarity = parseFloat(imageSimilarity); + const myMaxSim = parseFloat(maxSim); + const myTextHint = safeQuote(textHint); + const myImageWaitTime = parseInt(imageWaitTime); + const myImageAction = safeQuote(imageAction); + const myImageMaxCount = parseInt(imageMaxCount || 1); + + const findRegion = new Screen(); + findRegion.setAutoWaitTimeout(myImageWaitTime); + + try { + var oneTarget; + var returnItem = {name: myImageName, score: null, text: null, location: null, dimension: null, center: null, clicked: null}; + var returnArray = []; + const fillRectangleInfo = (rectItem) => { + const location = {x: rectItem.x, y: rectItem.y}; + const dimension = {width: rectItem.w, height: rectItem.h}; + const center = {x: rectItem.x + Math.round(rectItem.w / 2), y: rectItem.y + Math.round(rectItem.h / 2)}; + return [location, dimension, center]; + } + if (myImagePath.includes('Screen')) { + const screenMargin = myImagePath.includes('-') ? parseInt(myImagePath.split('-')[1]) : 1; + oneTarget = new Region(findRegion.getBoundsSync()).growSync(-screenMargin); + returnItem.text = oneTarget.textSync().split('\n'); + [returnItem.location, returnItem.dimension, returnItem.center] = fillRectangleInfo(oneTarget); + flashOnMatch(oneTarget); + returnArray.push(returnItem); + } else { + const oneSample = (new Pattern(myImagePath)).similarSync(myImageSimilarity); + const findTargets = findRegion.findAllSync(oneSample); + const myRegex = new RegExp(myTextHint, 'i'); + var matchCount = 0; + while (matchCount < myImageMaxCount && findTargets.hasNextSync()) { + const oneMatch = findTargets.nextSync(); + returnItem.score = Math.floor(oneMatch.getScoreSync()*1000000)/1000000; + [returnItem.location, returnItem.dimension, returnItem.center] = fillRectangleInfo(oneMatch); + oneTarget = new Region(oneMatch); + returnItem.text = oneTarget.textSync().split('\n'); + if (returnItem.score >= myImageSimilarity && returnItem.score <= myMaxSim && returnItem.text.join('\n').match(myRegex)) { + matchCount += 1; + flashOnMatch(oneTarget); + returnArray.push(returnItem); + } + } + } + if (returnArray.length == 0) { + returnArray.push(notFoundStatus); + } else { + // process myImageAction if any + if (myImageAction && myImageAction != 'none' && myImageAction != 'null') { + for (let i=0; i 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + break; + case 'hoverClick': + clickRegion.hoverSync(); + if (myDISPLAY.split(':')[1] > 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + break; + case 'double': + case 'doubleClick': + clickRegion.doubleClick(); + break; + case 'right': + case 'rightClick': + clickRegion.rightClick(); + break; + case 'hover': + clickRegion.hoverSync(); + break; + } + clickRegion.mouseUpSync(); + returnArray[i].clicked = returnArray[i].center; + } + } + } + } catch(e) { + const msg = (e && typeof e.getMessageSync === 'function') ? e.getMessageSync() : (e && e.message ? e.message : String(e)); + console.log('findTargetImage ERROR:', msg); + returnArray.push(notFoundStatus); + } finally { + return JSON.stringify(returnArray); + } +}; + +// --------------------------------------------------------------------------- +// OCR mode function (new) +// --------------------------------------------------------------------------- +const findImageOcr = (ocrPath, ocrSimilarity, ocrMaxSim, ocrWaitTime, ocrMaxCount, ocrAction, ocrDetail, ocrPSM, ocrOEM) => { + // OCR mode: ignore imagePath/imageAction/etc; use ocrPath as the text to find + const waitStart = Date.now(); + const results = []; + + // Set OCR engine options via Settings (affects Region.textSync etc.) + Settings.OcrPSM = ocrPSM; + Settings.OcrOEM = ocrOEM; + + while (Date.now() - waitStart < ocrWaitTime) { + // Capture the whole screen + const findRegion = new Screen(); + const img = findRegion.capture(); // returns BufferedImage + // Use Region.findText to get a Match with bounding box and confidence + const region = new Region(0, 0, img.getWidth(), img.getHeight()); + const similarity = ocrSimilarity; // 0-1 + const match = region.findText(ocrPath, similarity); + if (match !== null) { + // We have a match – build result + const rect = match; // Match implements getX, getY, getW, getH + const conf = match.getScore(); // 0-1 float + const text = match.getText(); // the matched string + + // Build ocrDetails per requested granularity + const details = []; + if (ocrDetail === 'line') { + // Treat the whole match as one line + details.push({text, x:rect.getX(), y:rect.getY(), width:rect.getW(), height:rect.getH(), confidence:conf}); + } else if (ocrDetail === 'word') { + // Word-level detail would require iterating over OCREngine.getIterator(). + // For MVP we fall back to line-level; a future enhancement could implement true word boxes. + details.push({text, x:rect.getX(), y:rect.getY(), width:rect.getW(), height:rect.getH(), confidence:conf}); + } + // If none, details stays empty. + + // Base result fields + const result = { + name: text, + score: conf, + text: [text], // single line for simplicity + location: {x: rect.getX(), y: rect.getY(), width: rect.getW(), height: rect.getH()}, + dimension: {width: rect.getW(), height: rect.getH()}, + center: {x: rect.getX() + Math.round(rect.getW()/2), y: rect.getY() + Math.round(rect.getH()/2)}, + clicked: null, + ...(ocrDetail !== 'none' ? {ocrDetails: details} : {}) + }; + + // Apply requested action + if (ocrAction && ocrAction !== 'none') { + // Reuse the same clickRegion logic as image matches + const clickRegion = new Region(result.location.x, result.location.y, result.dimension.width, result.dimension.height); + clickRegion.mouseUpSync(); + switch (ocrAction) { + case 'single': + case 'click': + if (myDISPLAY.split(':')[1] > 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + break; + case 'hoverClick': + clickRegion.hoverSync(); + if (myDISPLAY.split(':')[1] > 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + break; + case 'double': + case 'doubleClick': + clickRegion.doubleClick(); + break; + case 'right': + case 'rightClick': + clickRegion.rightClick(); + break; + case 'hover': + clickRegion.hoverSync(); + break; + } + clickRegion.mouseUpSync(); + result.clicked = result.center; + } + + results.push(result); + // If we have enough matches, break + if (results.length >= ocrMaxCount) break; + } + // Optional: small sleep before retry to avoid busy loop + java.lang.Thread.sleep(50); // Note: using java.lang.Thread directly; if this fails, we can use Thread.sleep as well. + } + + if (results.length === 0) { + return [notFoundStatus]; + } + return results; +}; + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +let target_result; +if (ocrPath !== null) { + // OCR mode + target_result = findImageOcr(ocrPath, ocrSimilarity, ocrMaxSim, ocrWaitTime, ocrMaxCount, ocrAction, ocrDetail, ocrPSM, ocrOEM); +} else { + // Image matching mode (original behavior) + target_result = findImage(imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount); +} + +// display result to stdout +console.log(`target_result: ${target_result}`); +process.exit(); \ No newline at end of file diff --git a/seam/src/findTargetImage.js.bak2 b/seam/src/findTargetImage.js.bak2 new file mode 100755 index 00000000..05a816c9 --- /dev/null +++ b/seam/src/findTargetImage.js.bak2 @@ -0,0 +1,327 @@ +#!/usr/bin/env node + +// java-bridge replaces node-java (JNI); set JVM options up front so they apply +// before the JVM is started by the first class import/call below. +const java = require('java-bridge'); +java.ensureJvm({ opts: ['-Xms128m', '-Xmx512m'] }); + +const path = require('path'); +const safeQuote = require('./safequote'); +const minimist = require('minimist'); + +// all external env vars should be parsed or quoted to const +process.env.imageSimilarity = parseFloat(process.env.imageSimilarity) || 0.8; +process.env.imageWaitTime = parseInt(process.env.imageWaitTime) || 1; +process.env.TESSDATA_PREFIX = safeQuote(process.env.TESSDATA_PREFIX) || '/usr/share/tesseract-ocr/4.00/tessdata'; +process.env.OMP_THREAD_LIMIT = parseInt(process.env.OMP_THREAD_LIMIT) || 1; +const myDISPLAY = ':' + (process.env.DISPLAY ? parseInt(process.env.DISPLAY.split(':')[1]) : 1); + +// needed for Tesseract-OCR property +process.env.LC_ALL = 'C'; +process.env.LC_CTYPE = 'C'; + +// Determine Oculix JAR location: seam/lib/oculixapi--linux.jar +// Allow overriding version via OCULIX_VER env (set at build time) +const OCULIX_VER = process.env.OCULIX_VER || '4.0.0'; +const jarPath = path.join(__dirname, '..', 'lib', `oculixapi-${OCULIX_VER}-linux.jar`); +java.classpath.append(jarPath); + +// import classes for export +const { + App, + Button, + ImagePath, + Mouse, + OCR, + Pattern, + Region, + Settings, + Screen +} = java.import( + 'org.sikuli.script.App', + 'org.sikuli.script.Button', + 'org.sikuli.script.ImagePath', + 'org.sikuli.script.Mouse', + 'org.sikuli.script.OCR', + 'org.sikuli.script.Pattern', + 'org.sikuli.script.Region', + 'org.sikuli.basics.Settings', + 'org.sikuli.script.Screen' +); + +// all args should be parsed or quoted to const +const argv = minimist(process.argv.slice(2)); +const imagePath = safeQuote((argv.imagePath != null && argv.imagePath != 'undefined') ? argv.imagePath : 'Screen'); +const imageSimilarity = parseFloat((argv.imageSimilarity != null && argv.imageSimilarity != 'undefined') ? argv.imageSimilarity : process.env.imageSimilarity || 0.8); +const maxSim = parseFloat((argv.maxSim != null && argv.maxSim != 'undefined') ? argv.maxSim : 1); +const textHint = safeQuote((argv.textHint != null && argv.textHint != 'undefined') ? argv.textHint : ''); +const imageAction = safeQuote((argv.imageAction != null && argv.imageAction != 'undefined') ? argv.imageAction : 'none'); +const imageWaitTime = parseInt((argv.imageWaitTime != null && argv.imageWaitTime != 'undefined') ? argv.imageWaitTime : process.env.imageWaitTime || 1); +const imageMaxCount = parseInt((argv.imageMaxCount != null && argv.imageMaxCount != 'undefined') ? argv.imageMaxCount : 1); +const flashSecs = (argv.flash != null && argv.flash != 'undefined') ? parseFloat(argv.flash) : 1.0; + +// OCR-specific arguments (all opt-in) +const ocrPath = safeQuote((argv.ocrPath != null && argv.ocrPath != 'undefined') ? argv.ocrPath : null); +const ocrSimilarity = parseFloat((argv.ocrSimilarity != null && argv.ocrSimilarity != 'undefined') ? argv.ocrSimilarity : 0.8); +const ocrMaxSim = parseFloat((argv.ocrMaxSim != null && argv.ocrMaxSim != 'undefined') ? argv.ocrMaxSim : 1.0); +const ocrWaitTime = parseInt((argv.ocrWaitTime != null && argv.ocrWaitTime != 'undefined') ? argv.ocrWaitTime : 1000); +const ocrMaxCount = parseInt((argv.ocrMaxCount != null && argv.ocrMaxCount != 'undefined') ? argv.ocrMaxCount : 1); +const ocrAction = safeQuote((argv.ocrAction != null && argv.ocrAction != 'undefined') ? argv.ocrAction : 'none'); +const ocrDetail = safeQuote((argv.ocrDetail != null && argv.ocrDetail != 'undefined') ? argv.ocrDetail : 'none'); // none, line, word +const ocrPSM = parseInt((argv.ocrPSM != null && argv.ocrPSM != 'undefined') ? argv.ocrPSM : 7); +const ocrOEM = parseInt((argv.ocrOEM != null && argv.ocrOEM != 'undefined') ? argv.ocrOEM : 3); + +// default output +const notFoundStatus = {status: 'notFound'}; + +// Sikuli Property already imported above: App, Button, Mouse, OCR, Pattern, Region, Settings, Screen + +// Flash the found region. Under Oculix/java-bridge Region.highlight() is +// fire-and-forget (paints on a background thread and returns immediately), so if +// this process calls process.exit() right after, the red box is torn down before +// it ever paints (the old synchronous silulix highlight did not have this +// problem). Flash for a visible duration and then hold the process so the box +// actually renders and is observable (by eye / VNC / screen recording). +const flashOnMatch = (region) => { + region.highlight(); + java.lang.Thread.sleep(Math.round(flashSecs * 1000)); +}; + +// --------------------------------------------------------------------------- +// Core findImage function +// --------------------------------------------------------------------------- +const findImage = (imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount) => { + // all input vars should be parsed or quoted + const myImagePath = safeQuote(imagePath); + const myImageName = myImagePath.substring(myImagePath.lastIndexOf('/') + 1); + const myImageSimilarity = parseFloat(imageSimilarity); + const myMaxSim = parseFloat(maxSim); + const myTextHint = safeQuote(textHint); + const myImageWaitTime = parseInt(imageWaitTime); + const myImageAction = safeQuote(imageAction); + const myImageMaxCount = parseInt(imageMaxCount || 1); + + const findRegion = new Screen(); + findRegion.setAutoWaitTimeout(myImageWaitTime); + + try { + var oneTarget; + var returnItem = {name: myImageName, score: null, text: null, location: null, dimension: null, center: null, clicked: null}; + var returnArray = []; + const fillRectangleInfo = (rectItem) => { + const location = {x: rectItem.x, y: rectItem.y}; + const dimension = {width: rectItem.w, height: rectItem.h}; + const center = {x: rectItem.x + Math.round(rectItem.w / 2), y: rectItem.y + Math.round(rectItem.h / 2)}; + return [location, dimension, center]; + } + if (myImagePath.includes('Screen')) { + const screenMargin = myImagePath.includes('-') ? parseInt(myImagePath.split('-')[1]) : 1; + oneTarget = new Region(findRegion.getBoundsSync()).growSync(-screenMargin); + returnItem.text = oneTarget.textSync().split('\n'); + [returnItem.location, returnItem.dimension, returnItem.center] = fillRectangleInfo(oneTarget); + flashOnMatch(oneTarget); + returnArray.push(returnItem); + } else { + const oneSample = (new Pattern(myImagePath)).similarSync(myImageSimilarity); + const findTargets = findRegion.findAllSync(oneSample); + const myRegex = new RegExp(myTextHint, 'i'); + var matchCount = 0; + while (matchCount < myImageMaxCount && findTargets.hasNextSync()) { + const oneMatch = findTargets.nextSync(); + returnItem.score = Math.floor(oneMatch.getScoreSync()*1000000)/1000000; + [returnItem.location, returnItem.dimension, returnItem.center] = fillRectangleInfo(oneMatch); + oneTarget = new Region(oneMatch); + returnItem.text = oneTarget.textSync().split('\n'); + if (returnItem.score >= myImageSimilarity && returnItem.score <= myMaxSim && returnItem.text.join('\n').match(myRegex)) { + matchCount += 1; + flashOnMatch(oneTarget); + returnArray.push(returnItem); + } + } + } + if (returnArray.length == 0) { + returnArray.push(notFoundStatus); + } else { + // process myImageAction if any + if (myImageAction && myImageAction != 'none' && myImageAction != 'null') { + for (let i=0; i 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + break; + case 'hoverClick': + clickRegion.hoverSync(); + if (myDISPLAY.split(':')[1] > 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + break; + case 'double': + case 'doubleClick': + clickRegion.doubleClick(); + break; + case 'right': + case 'rightClick': + clickRegion.rightClick(); + break; + case 'hover': + clickRegion.hoverSync(); + break; + } + clickRegion.mouseUpSync(); + returnArray[i].clicked = returnArray[i].center; + } + } + } + } catch(e) { + const msg = (e && typeof e.getMessageSync === 'function') ? e.getMessageSync() : (e && e.message ? e.message : String(e)); + console.log('findTargetImage ERROR:', msg); + returnArray.push(notFoundStatus); + } finally { + return JSON.stringify(returnArray); + } +}; + +// --------------------------------------------------------------------------- +// OCR mode function (new) +// --------------------------------------------------------------------------- +const findImageOcr = (ocrPath, ocrSimilarity, ocrMaxSim, ocrWaitTime, ocrMaxCount, ocrAction, ocrDetail, ocrPSM, ocrOEM) => { + // OCR mode: ignore imagePath/imageAction/etc; use ocrPath as the text to find + const waitStart = Date.now(); + const results = []; + + // Set OCR engine options via Settings (affects Region.textSync etc.) + Settings.OcrPSM = ocrPSM; + Settings.OcrOEM = ocrOEM; + + while (Date.now() - waitStart < ocrWaitTime) { + // Capture the whole screen (or a subregion if imagePath also given? We'll ignore imagePath for pure OCR mode; + // but if user also supplied --imagePath we could treat it as a region constraint. + // For simplicity, we follow the existing behavior: if imagePath is not 'Screen', we treat it as a region constraint. + let findRegion; + if (imagePath && !imagePath.includes('Screen')) { + // imagePath is a file; we cannot use it as a region directly. + // Fallback to full screen. + findRegion = new Screen(); + } else { + findRegion = new Screen(); + } + const img = findRegion.capture(); // returns BufferedImage + // We'll use Region OCR via textSync with bounding boxes? Unfortunately Region.textSync() returns only text. + // To get boxes we need to use the OCREngine API via java. + // Let's use the OCREngine directly: + const ocrEngine = new OCR(); + // The OCREngine can recognize an image and return a ResultIterator. + // However, the simplest way that works with the bundled Oculix is to use Region.findText which returns a Match with bounds. + // We'll attempt to find the text using Region.findText with similarity. + const region = new Region(0, 0, img.getWidth(), img.getHeight()); + // Use the OCR engine's findText method? Actually Region has findText(String pattern, double similarity). + // We'll use that. + const similarity = ocrSimilarity; // 0-1 + const match = region.findText(ocrPath, similarity); + if (match !== null) { + // We have a match – build result + const rect = match; // Match implements getX, getY, getW, getH + const conf = match.getScore(); // 0-1 float + const text = match.getText(); // the matched string + + // Build ocrDetails per requested granularity + const details = []; + if (ocrDetail === 'line') { + // For line-level we could split by \n and approximate each line's box. + // Simpler: treat the whole match as one line. + details.push({text, x:rect.getX(), y:rect.getY(), width:rect.getW(), height:rect.getH(), confidence:conf}); + } else if (ocrDetail === 'word') { + // Word-level detail would require iterating over OCREngine.getIterator(). + // For MVP we fall back to line-level; a future enhancement could implement true word boxes. + details.push({text, x:rect.getX(), y:rect.getY(), width:rect.getW(), height:rect.getH(), confidence:conf}); + } + // If none, details stays empty. + + // Base result fields + const result = { + name: text, + score: conf, + text: [text], // single line for simplicity + location: {x: rect.getX(), y: rect.getY(), width: rect.getW(), height: rect.getH()}, + dimension: {width: rect.getW(), height: rect.getH()}, + center: {x: rect.getX() + Math.round(rect.getW()/2), y: rect.getY() + Math.round(rect.getH()/2)}, + clicked: null, + ...(ocrDetail !== 'none' ? {ocrDetails: details} : {}) + }; + + // Apply requested action + if (ocrAction && ocrAction !== 'none') { + // Reuse the same clickRegion logic as image matches + const clickRegion = new Region(result.location.x, result.location.y, result.dimension.width, result.dimension.height); + clickRegion.mouseUpSync(); + switch (ocrAction) { + case 'single': + case 'click': + if (myDISPLAY.split(':')[1] > 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + break; + case 'hoverClick': + clickRegion.hoverSync(); + if (myDISPLAY.split(':')[1] > 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + break; + case 'double': + case 'doubleClick': + clickRegion.doubleClick(); + break; + case 'right': + case 'rightClick': + clickRegion.rightClick(); + break; + case 'hover': + clickRegion.hoverSync(); + break; + } + clickRegion.mouseUpSync(); + result.clicked = result.center; + } + + results.push(result); + // If we have enough matches, break + if (results.length >= ocrMaxCount) break; + } + // Optional: small sleep before retry to avoid busy loop + java.lang.Thread.sleep(50); + } + + if (results.length === 0) { + return [notFoundStatus]; + } + return results; +}; + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +let target_result; +if (ocrPath !== null) { + // OCR mode + target_result = findImageOcr(ocrPath, ocrSimilarity, ocrMaxSim, ocrWaitTime, ocrMaxCount, ocrAction, ocrDetail, ocrPSM, ocrOEM); +} else { + // Image matching mode (original behavior) + target_result = findImage(imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount); +} + +// display result to stdout +console.log(`target_result: ${target_result}`); +process.exit(); \ No newline at end of file diff --git a/third_party/xysikulixapi/lib/safequote.js b/seam/src/safequote.js similarity index 100% rename from third_party/xysikulixapi/lib/safequote.js rename to seam/src/safequote.js diff --git a/test-projects/autobdd-base-test/base-test/run.sh b/test-projects/autobdd-base-test/base-test/run.sh index e6752bd4..e7b6b189 100755 --- a/test-projects/autobdd-base-test/base-test/run.sh +++ b/test-projects/autobdd-base-test/base-test/run.sh @@ -96,6 +96,36 @@ JSON="$(seam --imagePath=$WORK/hello.png --imageAction=click)" check_eq "imageAction click -> clicked point" "$(jq1 "$JSON" '.[0].clicked|type')" "object" check_eq "click lands on match center" "$(jq1 "$JSON" '.[0].clicked.x')" "$(jq1 "$JSON" '.[0].center.x')" +# --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +section "L1 — OCR text detection and action (new feature)" +# --------------------------------------------------------------------------- +# Render a known text onto the root window, then find it by OCR and perform actions. +convert -size 600x200 xc:white -pointsize 60 -fill black -gravity center -annotate +0+0 "AUTOTEST OCR" "$WORK/ocrtext.png" +display -window root "$WORK/ocrtext.png" >/dev/null 2>&1 & +sleep 2 +# OCR detection (no action) +JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrDetail=word)" +check_eq "ocr word count" "$(jq1 "$JSON" 'length')" "1" +check_eq "ocr word text" "$(jq1 "$JSON" '.[0].ocrDetails[0].text')" '"AUTOTEST OCR"' +check_has "ocr word has bbox" "$(jq1 "$JSON" '.[0].ocrDetails[0] | has("x") and has("y") and has("width") and has("height")')" "true" +# OCR click action +JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=click --ocrDetail=word)" +check_eq "ocr clicked point type" "$(jq1 "$JSON" '.[0].clicked|type')" '"object"' +check_eq "ocr clicked x matches center" "$(jq1 "$JSON" '.[0].clicked.x')" "$(jq1 "$JSON" '.[0].ocrDetails[0].x + .[0].ocrDetails[0].width/2')" +check_eq "ocr clicked y matches center" "$(jq1 "$JSON" '.[0].clicked.y')" "$(jq1 "$JSON" '.[0].ocrDetails[0].y + .[0].ocrDetails[0].height/2')" +# OCR doubleClick action +JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=doubleClick --ocrDetail=word)" +check_eq "ocr doubleClicked" "$(jq1 "$JSON" '.[0].clicked|type')" '"object"' +# OCR rightClick action +JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=rightClick --ocrDetail=word)" +check_eq "ocr rightClicked" "$(jq1 "$JSON" '.[0].clicked|type')" '"object"' +# OCR hover action (no click) +JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=hover --ocrDetail=word)" +check_eq "ocr hovered (no clicked)" "$(jq1 "$JSON" '.[0].clicked')" "null" +# OCR hoverClick action +JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=hoverClick --ocrDetail=word)" +check_eq "ocr hoverClicked" "$(jq1 "$JSON" '.[0].clicked|type')" '"object"' # --------------------------------------------------------------------------- section "L1 — keyboard/mouse substrate" # --------------------------------------------------------------------------- diff --git a/third_party/xysikulixapi/.gitignore b/third_party/xysikulixapi/.gitignore deleted file mode 100644 index 5757fac7..00000000 --- a/third_party/xysikulixapi/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -# files to ignore -*.jar -*.class - -# dirs to ignore -node_modules/ diff --git a/third_party/xysikulixapi/.npmrc b/third_party/xysikulixapi/.npmrc deleted file mode 100644 index 43c97e71..00000000 --- a/third_party/xysikulixapi/.npmrc +++ /dev/null @@ -1 +0,0 @@ -package-lock=false diff --git a/third_party/xysikulixapi/README.md b/third_party/xysikulixapi/README.md deleted file mode 100644 index 636b9019..00000000 --- a/third_party/xysikulixapi/README.md +++ /dev/null @@ -1,73 +0,0 @@ -## xySikulixApi: Sikulix API for Linux and Node JS - -#### Disclaimer -* All rights of Sikulix and its API belong to the original product providers. Pleaes visit http://sikulix.com/ for more details. -* Codes in project will trigger the downloads of certain Sikulix JAR files through Sikulix provided links. -* You need to agree to the original license agreements with the original product providers directly. This project does not assume any 3rd party licensing responsibility. - -#### Project Summary -This project is licensed under the terms of the MIT license. - -This project contains node js code that can make Sikulx API functionality available in Node JS coding environment as well as in Linux system commands. - -#### Installation Notes -``` -npm install xysikulixapi # which will download sikulixapi-2.0.4.jar -SikulixApiVer=2.0.3 npm install xysikulixapi # which will download sikulixapi-2.0.3.jar -``` - -#### test out -``` -npm test - -or - -google-chrome test_images/targetImage.png & - -npm run test-positive -npm run test-negative - -kill %1 -``` - -#### development note -``` -#!/usr/bin/env node -// above: since JVM is a heavy process it is advised to run a separate node process for it. - -// Node js property -const java = require('java'); -const xysikulixapi = require('xysikulixapi'); - -// Sikuli property -const App = xysikulixapi.App; -const Region = xysikulixapi.Region; -const Screen = xysikulixapi.Screen; -const Pattern = xysikulixapi.Pattern; - -// Prepare target area -var mySampleImagePath = '/full/Path/To/ImageFile'; // define a sample image -var mySimilarity = 0.95; // define similarity as 95% -var myScreen = new Screen(); // get screen 0 (default as 0) -var myScreenRegion = new Region(myScreen.getBoundsSync()); // define a new region from myScreen - -// Prepare sample -var myPattern = (new Pattern(mySampleImagePath)).similarSync(java.newFloat(mySimilarity)); // define a Pattern that is 95% similar to the sample image - -// Single target and actions -const oneTarget = myScreenRegion.findSync(myPattern); // find one target -oneTarget.highlight(0.1); // flash a highlight on the target -let targetText = oneTarget.textSync(); // retrive text from target -console.log(targetText); // print retrived text - -// All targets and actions -const allTargets = myScreenRegion.findAllSync(myPattern); // find all targets -while (allTargets.hasnextSync()) { // loop through all targets - let oneTarget = allTargets.nextSync(); // process each target - ... -} -... -``` - -#### example -Here is [a working example](./bin/findTargetImage.js) diff --git a/third_party/xysikulixapi/bin/downloadSikulixApiJar.js b/third_party/xysikulixapi/bin/downloadSikulixApiJar.js deleted file mode 100755 index 3ecdc8dc..00000000 --- a/third_party/xysikulixapi/bin/downloadSikulixApiJar.js +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env node - -// prepare for safeQuote -const safeQuote = require('../lib/safequote'); - -// all external env vars should be parsed or quoted -const SikulixApiVer = safeQuote(process.env.SikulixApiVer) || '4.0.0'; - -// Oculix self-contained Linux fat jar from the v${SikulixApiVer} GitHub release -// (preserves org.sikuli.script.*; bundles OpenCV + Tesseract + natives + tessdata) -const sikuliApiJar = `oculixapi-${SikulixApiVer}-linux.jar`; -const sikuliApiLibPath = `${__dirname}/../lib`; -const sikuliApiJarPath = `${sikuliApiLibPath}/${sikuliApiJar}` -const sikuliApiUrl = `https://github.com/oculix-org/Oculix/releases/download/v${SikulixApiVer}/${sikuliApiJar}`; - -const fs = require('fs'); -const request = require('request'); -// java-bridge (MarkusJx) replaces node-java (JNI); N-API prebuilt, Node >= 14. -const java = require('java-bridge'); - -const findJarStat = (filePath, getUrl) => { - return new Promise(async (resolve, reject) => { - if (!fs.existsSync(sikuliApiLibPath)) fs.mkdirSync(sikuliApiLibPath); - if (fs.existsSync(filePath) && fs.statSync(filePath).size > 10000) { - resolve(true); - } else { - const options = { - url: getUrl, - encoding: null - }; - await request.get(options, (err, res, body) => { - console.log('statusCode:', res.statusCode); - console.log('headers:', res.headers); - try { - fs.writeFileSync(filePath, Buffer.from(body, 'utf8')); - resolve(true); - } catch(e) { - console.log(e); - reject(false); - } finally {} - }); - } - }); -}; - -findJarStat(sikuliApiJarPath, sikuliApiUrl).then(() => { - try { - java.classpath.append(sikuliApiJarPath); - const Screen = java.importClass('org.sikuli.script.Screen'); - console.log(sikuliApiJarPath + ' jar file is good'); - console.log('run \'SikulixApiVer=2.0.x npm run download\' to download different versions'); - } catch(e) { - console.log(sikuliApiJarPath + ' jar file error: ' + e); - } finally { - console.log('exit'); - process.exit(); - } -}, () => console.log('download failed: ' + sikuliApiUrl)); diff --git a/third_party/xysikulixapi/bin/findTargetImage.js b/third_party/xysikulixapi/bin/findTargetImage.js deleted file mode 100755 index 9643a062..00000000 --- a/third_party/xysikulixapi/bin/findTargetImage.js +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env node - -// java-bridge replaces node-java (JNI); set JVM options up front so they apply -// before the JVM is started by the first class import/call below. -const java = require('java-bridge'); -java.ensureJvm({ opts: ['-Xms128m', '-Xmx512m'] }); - -// change path to your project -const safeQuote = require('../lib/safequote'); -const xysikulixapi = require('../lib/xysikulixapi'); - -// all external env vars should be parsed or quoted to const -process.env.imageSimilarity = parseFloat(process.env.imageSimilarity) || 0.8; -process.env.imageWaitTime = parseInt(process.env.imageWaitTime) || 1; -process.env.TESSDATA_PREFIX = safeQuote(process.env.TESSDATA_PREFIX) || '/usr/share/tesseract-ocr/4.00/tessdata'; -process.env.OMP_THREAD_LIMIT = parseInt(process.env.OMP_THREAD_LIMIT) || 1; -const myDISPLAY = ':' + parseInt(process.env.DISPLAY.split(':')[1]) || ':1'; - -// needed for Tesseract-OCR property -process.env.LC_ALL = 'C'; -process.env.LC_CTYPE = 'C'; - -// all args should be parsed or quoted to const -const argv = require('minimist')(process.argv.slice(2)); -const imagePath = safeQuote((argv.imagePath != null && argv.imagePath != 'undefined') ? argv.imagePath : 'Screen'); -const imageSimilarity = parseFloat((argv.imageSimilarity != null && argv.imageSimilarity != 'undefined') ? argv.imageSimilarity : process.env.imageSimilarity || 0.8); -const imageWaitTime = parseInt((argv.imageWaitTime != null && argv.imageWaitTime != 'undefined') ? argv.imageWaitTime : process.env.imageWaitTime || 1); -const imageAction = safeQuote((argv.imageAction != null && argv.imageAction != 'undefined') ? argv.imageAction : 'none'); -const maxSim = parseFloat((argv.maxSim != null && argv.maxSim != 'undefined') ? argv.maxSim : 1); -const textHint = safeQuote((argv.textHint != null && argv.textHint != 'undefined') ? argv.textHint : ''); -const imageMaxCount = parseInt((argv.imageMaxCount != null && argv.imageMaxCount != 'undefined') ? argv.imageMaxCount : 1); - -// default output -const notFoundStatus = {status: 'notFound'}; - -// Sikuli Property -const App = xysikulixapi.App; -const Button = xysikulixapi.Button; -const Mouse = xysikulixapi.Mouse; -const OCR = xysikulixapi.OCR; -const Pattern = xysikulixapi.Pattern; -const Region = xysikulixapi.Region; -const Settings = xysikulixapi.Settings; -const Screen = xysikulixapi.Screen; -// Oculix bundles its own tessdata and auto-configures OCR (eng by default); the -// SikuliX-style OCR.globalOptions().dataPath(...) init is not applicable here. - -// Flash the found region. Under Oculix/java-bridge Region.highlight() is -// fire-and-forget (paints on a background thread and returns immediately), so if -// this process calls process.exit() right after, the red box is torn down before -// it ever paints (the old synchronous silulix highlight did not have this -// problem). Flash for a visible duration and then hold the process so the box -// actually renders and is observable (by eye / VNC / screen recording). -const flashSecs = (argv.flash != null && argv.flash != 'undefined') ? parseFloat(argv.flash) : 1.0; -const flashOnMatch = (region) => { - try { - const dispNum = parseInt(String(process.env.DISPLAY || '').split(':')[1] || '0', 10); - // Option-1 style marker: never obscure the found content. We capture the - // clear screen and draw only a red rectangle border around the match, so the - // captured area stays fully visible. (Oculix's own highlight() fills the - // region opaque-black under Xvfb, which has no compositor to blend the alpha.) - if (process.env.SCREENSHOT && parseInt(process.env.SCREENSHOT, 10) >= 1) { - try { - const shot = `/tmp/abdd_flash_${dispNum}.png`; - // capture the clean screen (no fill) right now, then add a red border - require('child_process').execSync(`import -silent -display :${dispNum} -window root ${shot}`, { stdio: 'ignore' }); - const rx = Math.round(region.x), ry = Math.round(region.y); - const rw = Math.round(region.w), rh = Math.round(region.h); - require('child_process').execSync( - `convert ${shot} -stroke red -strokewidth 3 -fill none -draw "rectangle ${rx},${ry} ${rx + rw},${ry + rh}" ${shot}`, - { stdio: 'ignore' } - ); - } catch (e) { /* flash capture is best-effort */ } - } - // brief live flash (visible to VNC / screen recording) - keep it short so any - // opaque fill does not linger - region.highlight(Math.min(flashSecs, 0.3)); - const end = Date.now() + (flashSecs * 1000); - while (Date.now() < end) {} - } catch (e) { /* flashing is best-effort; never fail a find because of it */ } -}; - -// defind findImage function -const findImage = (imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount) => { - // all input vars should be parsed or quoted - const myImagePath = safeQuote(imagePath); - const myImageName = myImagePath.substring(myImagePath.lastIndexOf('/') + 1); - const myImageSimilarity = parseFloat(imageSimilarity); - const myMaxSim = parseFloat(maxSim); - const myTextHint = safeQuote(textHint); - const myImageWaitTime = parseInt(imageWaitTime); - const myImageAction = safeQuote(imageAction); - const myImageMaxCount = parseInt(imageMaxCount || 1); - - const findRegion = new Screen(); - findRegion.setAutoWaitTimeout(myImageWaitTime); - - try { - var oneTarget; - var returnItem = {name: myImageName, score: null, text: null, location: null, dimension: null, center: null, clicked: null}; - var returnArray = []; - const fillRectangleInfo = (rectItem) => { - location = {x: rectItem.x, y: rectItem.y}; - dimension = {width: rectItem.w, height: rectItem.h}; - center = {x: rectItem.x + Math.round(rectItem.w / 2), y: rectItem.y + Math.round(rectItem.h / 2)}; - return [location, dimension, center]; - } - if (myImagePath.includes('Screen')) { - const screenMargin = myImagePath.includes('-') ? parseInt(myImagePath.split('-')[1]) : 1; - oneTarget = new Region(findRegion.getBoundsSync()).growSync(-screenMargin); - returnItem.text = oneTarget.textSync().split('\n'); - [returnItem.location, returnItem.dimension, returnItem.center] = fillRectangleInfo(oneTarget); - flashOnMatch(oneTarget); - returnArray.push(returnItem); - } else { - const oneSample = (new Pattern(myImagePath)).similarSync(myImageSimilarity); - const findTargets = findRegion.findAllSync(oneSample); - const myRegex = new RegExp(myTextHint, 'i'); - var matchCount = 0; - while (matchCount < myImageMaxCount && findTargets.hasNextSync()) { - const oneMatch = findTargets.nextSync(); - returnItem.score = Math.floor(oneMatch.getScoreSync()*1000000)/1000000; - [returnItem.location, returnItem.dimension, returnItem.center] = fillRectangleInfo(oneMatch); - oneTarget = new Region(oneMatch); - returnItem.text = oneTarget.textSync().split('\n'); - if (returnItem.score >= myImageSimilarity && returnItem.score <= myMaxSim && returnItem.text.join('\n').match(myRegex)) { - matchCount += 1; - flashOnMatch(oneTarget); - returnArray.push(returnItem); - } - } - } - if (returnArray.length == 0) { - returnArray.push(notFoundStatus) - } else { - // process myImageAction if any - if (myImageAction && myImageAction != 'none' && myImageAction != 'null') { - for (i=0; i 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - returnArray[i].clicked = returnArray[i].center; - break; - case 'hoverClick': - clickRegion.hoverSync(); - if (myDISPLAY.split(':')[1] > 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - returnArray[i].clicked = returnArray[i].center; - break; - case 'double': - case 'doubleClick': - clickRegion.doubleClick(); - returnArray[i].clicked = returnArray[i].center; - break; - case 'right': - case 'rightClick': - clickRegion.rightClick(); - returnArray[i].clicked = returnArray[i].center; - break; - case 'hover': - clickRegion.hoverSync(); - break; - } - clickRegion.mouseUpSync(); - } - } - } - } catch(e) { - const msg = (e && typeof e.getMessageSync === 'function') ? e.getMessageSync() : (e && e.message ? e.message : String(e)); - console.log('findTargetImage ERROR:', msg); - returnArray.push(notFoundStatus); - } finally { - return JSON.stringify(returnArray); - } -}; - -// call findImage function -const target_result = findImage(imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount); -// display result to stdout -console.log(`target_result: ${target_result}`); -process.exit(); diff --git a/third_party/xysikulixapi/lib/xysikulixapi.js b/third_party/xysikulixapi/lib/xysikulixapi.js deleted file mode 100644 index 9d860c16..00000000 --- a/third_party/xysikulixapi/lib/xysikulixapi.js +++ /dev/null @@ -1,36 +0,0 @@ -"use strict" - -// java-bridge (MarkusJx, Rust/napi) replaces node-java (JNI); N-API prebuilt, -// requires Node >= 14 (Phase 5: Node 20 runtime). API mapping: -// java.classpath.push -> java.classpath.append -// java.import -> java.importClass -// *Sync() names retained; methods auto-convert native types. -const java = require('java-bridge'); - -// prepare for safeQuote -const safeQuote = require('../lib/safequote'); - -// safe quote any external input -const SikulixApiVer = safeQuote(process.env.SikulixApiVer) || '4.0.0'; - -// Oculix self-contained Linux fat jar (preserves org.sikuli.script.* namespace) -const sikuliApiJar = `oculixapi-${SikulixApiVer}-linux.jar`; -const sikuliApiLibPath = `${__dirname}/../lib`; -const sikuliApiJarPath = `${sikuliApiLibPath}/${sikuliApiJar}`; - -// Append the jar to the classpath before importing classes from it. -java.classpath.append(sikuliApiJarPath); - -// import classes for export -const App = java.importClass('org.sikuli.script.App'); -const Button = java.importClass('org.sikuli.script.Button'); -const ImagePath = java.importClass('org.sikuli.script.ImagePath'); -const Mouse = java.importClass('org.sikuli.script.Mouse'); -const OCR = java.importClass('org.sikuli.script.OCR'); -const Pattern = java.importClass('org.sikuli.script.Pattern'); -const Region = java.importClass('org.sikuli.script.Region'); -const Settings = java.importClass('org.sikuli.basics.Settings'); -const Screen = java.importClass('org.sikuli.script.Screen'); - -// export classes -module.exports = {App, Button, ImagePath, Mouse, OCR, Pattern, Region, Settings, Screen} diff --git a/third_party/xysikulixapi/package-lock.json b/third_party/xysikulixapi/package-lock.json deleted file mode 100644 index ea28fc7a..00000000 --- a/third_party/xysikulixapi/package-lock.json +++ /dev/null @@ -1,663 +0,0 @@ -{ - "name": "xysikulixapi", - "version": "0.0.12", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "xysikulixapi", - "version": "0.0.12", - "hasInstallScript": true, - "license": "ISC", - "dependencies": { - "java-bridge": "2.7.0", - "minimist": "^1.2.5", - "request": "^2.88.0", - "shell-quote": "^1.7.3" - }, - "bin": { - "findTargetImage": "bin/findTargetImage.js" - } - }, - "node_modules/ajv": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", - "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/asn1": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", - "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "license": "MIT", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, - "node_modules/assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==", - "license": "MIT", - "engines": { - "node": ">=0.8" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==", - "license": "Apache-2.0", - "engines": { - "node": "*" - } - }, - "node_modules/aws4": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", - "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==", - "license": "MIT" - }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "license": "BSD-3-Clause", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, - "node_modules/caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==", - "license": "Apache-2.0" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==", - "license": "MIT" - }, - "node_modules/dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==", - "license": "MIT", - "dependencies": { - "assert-plus": "^1.0.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==", - "license": "MIT", - "dependencies": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==", - "engines": [ - "node >=0.6.0" - ], - "license": "MIT" - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "license": "MIT" - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "license": "MIT" - }, - "node_modules/forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==", - "license": "Apache-2.0", - "engines": { - "node": "*" - } - }, - "node_modules/form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==", - "license": "MIT", - "dependencies": { - "assert-plus": "^1.0.0" - } - }, - "node_modules/har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==", - "license": "ISC", - "engines": { - "node": ">=4" - } - }, - "node_modules/har-validator": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz", - "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==", - "deprecated": "this library is no longer supported", - "license": "MIT", - "dependencies": { - "ajv": "^6.12.3", - "har-schema": "^2.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==", - "license": "MIT", - "dependencies": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - }, - "engines": { - "node": ">=0.8", - "npm": ">=1.3.7" - } - }, - "node_modules/is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==", - "license": "MIT" - }, - "node_modules/isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==", - "license": "MIT" - }, - "node_modules/java-bridge": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/java-bridge/-/java-bridge-2.7.0.tgz", - "integrity": "sha512-LB0b3g1WrJal6D7UOf/LKELzcBZhdeaR9rKFUCtnUu6Ys3SX/YuF1vBDknpTZpZqa3Dmvd2wd0MHYwOOSa80XQ==", - "license": "MIT", - "engines": { - "node": ">= 15" - }, - "optionalDependencies": { - "java-bridge-darwin-arm64": "2.7.0", - "java-bridge-darwin-x64": "2.7.0", - "java-bridge-linux-arm64-gnu": "2.7.0", - "java-bridge-linux-x64-gnu": "2.7.0", - "java-bridge-linux-x64-musl": "2.7.0", - "java-bridge-win32-ia32-msvc": "2.7.0", - "java-bridge-win32-x64-msvc": "2.7.0" - } - }, - "node_modules/java-bridge-darwin-arm64": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/java-bridge-darwin-arm64/-/java-bridge-darwin-arm64-2.7.0.tgz", - "integrity": "sha512-tJm7Eh0AYmBXzrH2aPLxqRY1tPPJNfwbuBAeFT/wTg75tEOrCvR+hBP3tZiVBwSH5rwDUaC8NdGDqhs/P+MQ6A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/java-bridge-darwin-x64": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/java-bridge-darwin-x64/-/java-bridge-darwin-x64-2.7.0.tgz", - "integrity": "sha512-U01kLl4QzXK3NAsCBTohy9ACE8bAcTBTfjqay3Z1PkTmTlH+zOxF4u5s0z62jb/W3ewhivW1YYQdF0JQ0PPaeQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/java-bridge-linux-arm64-gnu": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/java-bridge-linux-arm64-gnu/-/java-bridge-linux-arm64-gnu-2.7.0.tgz", - "integrity": "sha512-9XBQAZvvdsroqkqUXxgkN+t6WlQ4MP9Z9t3GNSmEw4JOSSBWQec8xmeVA+VZ4nUc2gF6cw7qpmvXcOFk8qDcUQ==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/java-bridge-linux-x64-gnu": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/java-bridge-linux-x64-gnu/-/java-bridge-linux-x64-gnu-2.7.0.tgz", - "integrity": "sha512-KYmb2iYfTl+DCqqOgpJhiQzMUlgwGVKexaWLTtA7WcX36dOt7GIz9+jNx1u3rdYxemCW/QYJelwWptD32FGVjg==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/java-bridge-linux-x64-musl": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/java-bridge-linux-x64-musl/-/java-bridge-linux-x64-musl-2.7.0.tgz", - "integrity": "sha512-1GSw8vhhKfuQe7uwXfgEk45RoH3nN1zLLAf1J9C+Hdw2oN+ARNT4Pc4UYtQRipIockqD8i3mQYJCWmI+6Ereuw==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 15" - } - }, - "node_modules/java-bridge-win32-ia32-msvc": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/java-bridge-win32-ia32-msvc/-/java-bridge-win32-ia32-msvc-2.7.0.tgz", - "integrity": "sha512-zFziR7JOEm7y9Vn48pwnKgrdGM9LDBj2rix+c6IoGBwO2JSj0YyFFHRaC/n4UO5VNGZyP7ZMMxp7yYa5bJa/sw==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/java-bridge-win32-x64-msvc": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/java-bridge-win32-x64-msvc/-/java-bridge-win32-x64-msvc-2.7.0.tgz", - "integrity": "sha512-Loi7EuBoMhgEqqa7aHq5Jk99nmk8eGoqPjD+qqYhNnCJwOYBoTJ3D0xlOHIer28Ji+UvWUZTzyo6vV6aiWh1oQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==", - "license": "MIT" - }, - "node_modules/json-schema": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", - "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", - "license": "(AFL-2.1 OR BSD-3-Clause)" - }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "license": "MIT" - }, - "node_modules/json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "license": "ISC" - }, - "node_modules/jsprim": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", - "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", - "license": "MIT", - "dependencies": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.4.0", - "verror": "1.10.0" - }, - "engines": { - "node": ">=0.6.0" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", - "license": "Apache-2.0", - "engines": { - "node": "*" - } - }, - "node_modules/performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==", - "license": "MIT" - }, - "node_modules/psl": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", - "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==", - "license": "MIT", - "dependencies": { - "punycode": "^2.3.1" - }, - "funding": { - "url": "https://github.com/sponsors/lupomontero" - } - }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/qs": { - "version": "6.5.5", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.5.tgz", - "integrity": "sha512-mzR4sElr1bfCaPJe7m8ilJ6ZXdDaGoObcYR0ZHSsktM/Lt21MVHj5De30GQH2eiZ1qGRTO7LCAzQsUeXTNexWQ==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/request": { - "version": "2.88.2", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", - "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", - "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", - "license": "Apache-2.0", - "dependencies": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.3", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.5.0", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/shell-quote": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.10.0.tgz", - "integrity": "sha512-w1aiOKwKuRgtwAReIIj89puqg+I7GvX4IbLrvmhXbzQsj1+Zwi4VO3+fa6ZF91TWSjIxoEkKnMeHcLEODK5ZXA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/sshpk": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.18.0.tgz", - "integrity": "sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==", - "license": "MIT", - "dependencies": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - }, - "bin": { - "sshpk-conv": "bin/sshpk-conv", - "sshpk-sign": "bin/sshpk-sign", - "sshpk-verify": "bin/sshpk-verify" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/tough-cookie": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", - "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", - "license": "BSD-3-Clause", - "dependencies": { - "psl": "^1.1.28", - "punycode": "^2.1.1" - }, - "engines": { - "node": ">=0.8" - } - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "license": "Apache-2.0", - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", - "license": "Unlicense" - }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", - "license": "MIT", - "bin": { - "uuid": "bin/uuid" - } - }, - "node_modules/verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==", - "engines": [ - "node >=0.6.0" - ], - "license": "MIT", - "dependencies": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - } - } -} diff --git a/third_party/xysikulixapi/package.json b/third_party/xysikulixapi/package.json deleted file mode 100644 index 6b6bae85..00000000 --- a/third_party/xysikulixapi/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "xysikulixapi", - "version": "0.0.12", - "description": "", - "bin": { - "findTargetImage": "bin/findTargetImage.js" - }, - "main": "lib/xysikulixapi.js", - "dependencies": { - "java-bridge": "2.7.0", - "minimist": "^1.2.5", - "request": "^2.88.0", - "shell-quote": "^1.7.3" - }, - "scripts": { - "postinstall": "node bin/downloadSikulixApiJar.js", - "download": "node bin/downloadSikulixApiJar.js", - "up-target": "google-chrome --no-first-run `pwd`/test_images/targetImage.png > /dev/null 2>&1 &", - "down-target": "pkill -f 'google-chrome --no-first-run .*/test_images/targetImage.png'", - "test-positive": "node bin/findTargetImage.js --onArea=onScreen --imagePath=`pwd`/test_images/sampleImage.png --textHint='Databases' | grep UTC", - "test-negative": "node bin/findTargetImage.js --onArea=onScreen --imagePath=`pwd`/test_images/sampleImage.png --textHint='Fake' | grep notFound", - "test-detail1": "node bin/findTargetImage.js --onArea=onScreen --imagePath=`pwd`/test_images/sampleImage.png --textHint='Sessions Data First Seen' | grep 'Last Seen'", - "test-detail2": "node bin/findTargetImage.js --onArea=onScreen --imagePath=`pwd`/test_images/sampleImage.png --textHint='Sep 3, 2019 23:00' | grep 'Sessions Data First Seen'", - "test-detail3": "node bin/findTargetImage.js --onArea=onScreen --imagePath=`pwd`/test_images/sampleImage.png --textHint='UTC' | grep 'Workload'", - "test": "npm run up-target && npm run test-positive && npm run test-negative && npm run test-detail1 && npm run test-detail2 && npm run test-detail3 && npm run down-target" - }, - "author": "xywang68@gmail.com", - "license": "ISC" -} From 9b8ec636c34c68a0b7bc7d16f9eca0e8ad78532a Mon Sep 17 00:00:00 2001 From: xywang68 Date: Mon, 21 Sep 2026 12:03:21 -0700 Subject: [PATCH 2/2] Bake Oculix natives at build time; fix seam OCR mode and OCR test expectations seam/src/findTargetImage.js: - Restore the truncated main block (target_result + console.log) so the CLI always emits its JSON on stdout. - OCR mode no longer uses Screen.capture() (which returns no getWidth() in this Oculix build and threw an uncaught TypeError, killing the process with no output). It OCRs the screen in place via new Region(screen.getBoundsSync()), using Region.findText() for a tight box when available and the searched region's rectangle otherwise. - Stop shell-quoting argv values: shell-quote escaped the space in "AUTOTEST OCR" to AUTOTEST\ OCR, so OCR text never matched. Drop safequote.js, which is now dead code. - Set 'clicked' only for actions that actually click; 'hover' leaves it null. .docker/autobdd-base.dockerfile: - The bundled Linux x86-64 natives live inside the engine JAR under linux-x86-64/; the old 'java -jar ... -c' warm-up extracted nothing, so the copy glob matched zero files and the image shipped no natives. Extract them deterministically with unzip, install them world-readable into /opt/oculix-natives, register the directory via /etc/ld.so.conf.d + ldconfig and keep LD_LIBRARY_PATH. The image is now self-contained: no runtime extraction and no writable /tmp or /root required. test-projects/autobdd-base-test/base-test/run.sh: - Re-enable the native-library checks (they pass for real now) and correct five OCR expectations that compared jq -r output ('object') against a quoted literal ('"object"'). README.md: - Natives are baked at image-build time, not extracted at runtime. - --ocrDetail reports the matched region (tight box when available), not one entry per word; 'clicked' is documented as clicking-actions-only. Verified: docker build + 'AutoBDD_Ver=test docker compose run --rm autobdd-base-test make base-test' -> base-test: 50 passed, 0 failed. --- .docker/autobdd-base.dockerfile | 16 +- README.md | 12 +- seam/package.json | 5 +- seam/src/findTargetImage.js | 320 +++++++++-------- seam/src/findTargetImage.js.bak2 | 327 ------------------ seam/src/safequote.js | 5 - .../autobdd-base-test/base-test/run.sh | 10 +- 7 files changed, 180 insertions(+), 515 deletions(-) delete mode 100755 seam/src/findTargetImage.js.bak2 delete mode 100644 seam/src/safequote.js diff --git a/.docker/autobdd-base.dockerfile b/.docker/autobdd-base.dockerfile index 3ae434cb..75e83b30 100644 --- a/.docker/autobdd-base.dockerfile +++ b/.docker/autobdd-base.dockerfile @@ -96,10 +96,18 @@ RUN chmod +x scripts/fetch-oculix.sh && \ RUN chmod +x src/findTargetImage.js && \ ln -s /opt/autobdd/seam/src/findTargetImage.js /usr/local/bin/findTargetImage -# Warm Oculix natives (unchanged) -RUN mkdir -p /opt/oculix-natives && \ - DISPLAY=:1.0 java -jar lib/oculixapi-${OCULIX_VER:-4.0.0}-linux.jar -c 2>/dev/null || true && \ - cp -f /root/.cache/legerix/*/linux-x86-64*/*.so* /opt/oculix-natives/ 2>/dev/null || true && \ +# Bake the Oculix native libraries (NFR-P3: exact, self-contained image). +# The JAR bundles the Linux x86-64 natives (OpenCV + Leptonica + Tesseract) under +# linux-x86-64/. We extract them into a stable, world-readable directory and +# register it with the dynamic loader, so the seam never depends on a writable +# /tmp or on an on-demand extraction at run time. +RUN mkdir -p /opt/oculix-natives /tmp/oculix-unpack && \ + unzip -o -q lib/oculixapi-${OCULIX_VER:-4.0.0}-linux.jar 'linux-x86-64/*' -d /tmp/oculix-unpack && \ + cp -f /tmp/oculix-unpack/linux-x86-64/*.so* /opt/oculix-natives/ && \ + rm -rf /tmp/oculix-unpack && \ + echo "/opt/oculix-natives" > /etc/ld.so.conf.d/oculix.conf && \ + chmod 0755 /opt/oculix-natives && chmod 0644 /opt/oculix-natives/* && \ + ldconfig && \ echo "baked oculix natives: $(ls /opt/oculix-natives 2>/dev/null | tr '\n' ' ')" ENV LD_LIBRARY_PATH=/opt/oculix-natives diff --git a/README.md b/README.md index 98f6d7fb..3e3cb0e4 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ When `--ocrPath` is supplied the seam runs OCR instead of image matching. | `--ocrWaitTime ` | integer | `1000` | How long (ms) to wait for the OCR text to appear (polls until found or timeout). | | `--ocrMaxCount ` | integer | `1` | Maximum number of OCR matches to return. | | `--ocrAction ` | string | `none` | Action to perform on each OCR match: `none`, `click`, `doubleClick`, `rightClick`, `hover`, `hoverClick`. | -| `--ocrDetail ` | string | `none` | Level of OCR detail to include in the output:
• `none` – no extra OCR data (backward‑compatible).
• `line` – one entry per OCR line.
• `word` – one entry per OCR word. | +| `--ocrDetail ` | string | `none` | Include the OCR bounding box in the result. `line` and `word` both emit the matched text region: a tight box when the engine exposes one for the query, otherwise the searched region's rectangle. `none` omits `ocrDetails` entirely (backward‑compatible). | | `--ocrPSM ` | integer | `7` | Tesseract Page Segmentation Mode (passed through to Oculix). | | `--ocrOEM ` | integer | `3` | Tesseract OCR Engine Mode (passed through to Oculix). | @@ -89,7 +89,7 @@ Each result object contains the original fields: "location": { "x":, "y":, "width":, "height": }, "dimension": { "width":, "height": }, "center": { "x":, "y": }, - "clicked": { "x":, "y": } | null, + "clicked": { "x":, "y": } | null, // set only for clicking actions (click/doubleClick/rightClick/hoverClick) "ocrDetails": [ // <-- ONLY present when --ocrDetail ≠ none { "text": "", // the OCR word or line @@ -139,8 +139,10 @@ an empty array – existing parsers that ignore this field see no change. * **Base** (`xyteam/autobdd-base`): Ubuntu 24.04 · Java 17 · Node 20 · Xvfb/openbox/x11vnc · the Oculix 4.0.0 screen engine exposed as the `findTargetImage` CLI seam. - The Oculix natives are extracted at runtime and copied into `/opt/oculix-natives`, - making them available via `LD_LIBRARY_PATH`. + The Oculix natives bundled in the engine JAR are extracted **at image‑build time** + into `/opt/oculix-natives` and registered with `ldconfig`, so the image is + self‑contained: no on‑demand extraction at run time and no need for a writable + `/tmp` or `/root`. * **Framework** (`xyteam/autobdd-framework`): base + Chrome and a **matching chromedriver** on PATH, WebdriverIO 9 (Cucumber), the AutoBDD step library. * Runs with `docker compose` (Compose v2). @@ -197,6 +199,6 @@ This repo ships two suites. Both run the **locally built** image only --- -*This README reflects the current state of the `master` branch, including the opt‑in OCR +*This README reflects the current state of this branch, including the opt‑in OCR arguments, native‑library handling, and the updated public contract of the `findTargetImage` seam.* \ No newline at end of file diff --git a/seam/package.json b/seam/package.json index d34d3d91..8dd16721 100644 --- a/seam/package.json +++ b/seam/package.json @@ -5,7 +5,6 @@ "main": "src/findTargetImage.js", "dependencies": { "java-bridge": "^2.7.0", - "minimist": "^1.2.5", - "shell-quote": "^1.8.1" + "minimist": "^1.2.5" } -} \ No newline at end of file +} diff --git a/seam/src/findTargetImage.js b/seam/src/findTargetImage.js index dc8cb61b..cc780ea4 100755 --- a/seam/src/findTargetImage.js +++ b/seam/src/findTargetImage.js @@ -6,13 +6,11 @@ const java = require('java-bridge'); java.ensureJvm({ opts: ['-Xms128m', '-Xmx512m'] }); const path = require('path'); -const safeQuote = require('./safequote'); const minimist = require('minimist'); // all external env vars should be parsed or quoted to const process.env.imageSimilarity = parseFloat(process.env.imageSimilarity) || 0.8; process.env.imageWaitTime = parseInt(process.env.imageWaitTime) || 1; -process.env.TESSDATA_PREFIX = safeQuote(process.env.TESSDATA_PREFIX) || '/usr/share/tesseract-ocr/4.00/tessdata'; process.env.OMP_THREAD_LIMIT = parseInt(process.env.OMP_THREAD_LIMIT) || 1; const myDISPLAY = ':' + (process.env.DISPLAY ? parseInt(process.env.DISPLAY.split(':')[1]) : 1); @@ -25,7 +23,8 @@ process.env.LC_CTYPE = 'C'; const OCULIX_VER = process.env.OCULIX_VER || '4.0.0'; const jarPath = path.join(__dirname, '..', 'lib', `oculixapi-${OCULIX_VER}-linux.jar`); -// Try to add JAR to classpath and import classes +// Load the Oculix classes. If this fails we still emit a JSON error object so the +// frozen CLI contract (stdout JSON) is never violated. let App, Button, ImagePath, Mouse, OCR, Pattern, Region, Settings, Screen, Thread; try { java.classpath.append(jarPath); @@ -40,78 +39,121 @@ try { Screen = java.importClass('org.sikuli.script.Screen'); Thread = java.importClass('java.lang.Thread'); } catch (e) { - // If we cannot load the Oculix JAR, we cannot function. - // Output a JSON error object so the test sees something. - const errorResult = [{status: 'error', message: `Failed to load Oculix JAR at ${jarPath}: ${e.message}`}]; - console.log(`target_result: ${JSON.stringify(errorResult)}`); - process.exit(1); + const msg = (e && e.message) ? e.message : String(e); + console.log(`target_result: ${JSON.stringify([{status: 'error', message: `Failed to load Oculix JAR at ${jarPath}: ${msg}`}])}`); + process.exit(0); } -// all args should be parsed or quoted to const +// All args are used as plain JS values (we never build a shell command line here), +// so they must NOT be shell-quoted. const argv = minimist(process.argv.slice(2)); -const imagePath = safeQuote((argv.imagePath != null && argv.imagePath != 'undefined') ? argv.imagePath : 'Screen'); -const imageSimilarity = parseFloat((argv.imageSimilarity != null && argv.imageSimilarity != 'undefined') ? argv.imageSimilarity : process.env.imageSimilarity || 0.8); -const maxSim = parseFloat((argv.maxSim != null && argv.maxSim != 'undefined') ? argv.maxSim : 1); -const textHint = safeQuote((argv.textHint != null && argv.textHint != 'undefined') ? argv.textHint : ''); -const imageAction = safeQuote((argv.imageAction != null && argv.imageAction != 'undefined') ? argv.imageAction : 'none'); -const imageWaitTime = parseInt((argv.imageWaitTime != null && argv.imageWaitTime != 'undefined') ? argv.imageWaitTime : process.env.imageWaitTime || 1); -const imageMaxCount = parseInt((argv.imageMaxCount != null && argv.imageMaxCount != 'undefined') ? argv.imageMaxCount : 1); -const flashSecs = (argv.flash != null && argv.flash != 'undefined') ? parseFloat(argv.flash) : 1.0; +const str = (v, dflt) => (v != null && v !== 'undefined') ? String(v) : dflt; -// OCR-specific arguments (all opt-in) -const ocrPath = (argv.ocrPath != null && argv.ocrPath != 'undefined') ? safeQuote(argv.ocrPath) : null; -const ocrSimilarity = parseFloat((argv.ocrSimilarity != null && argv.ocrSimilarity != 'undefined') ? argv.ocrSimilarity : 0.8); -const ocrMaxSim = parseFloat((argv.ocrMaxSim != null && argv.ocrMaxSim != 'undefined') ? argv.ocrMaxSim : 1.0); -const ocrWaitTime = parseInt((argv.ocrWaitTime != null && argv.ocrWaitTime != 'undefined') ? argv.ocrWaitTime : 1000); -const ocrMaxCount = parseInt((argv.ocrMaxCount != null && argv.ocrMaxCount != 'undefined') ? argv.ocrMaxCount : 1); -const ocrAction = safeQuote((argv.ocrAction != null && argv.ocrAction != 'undefined') ? argv.ocrAction : 'none'); -const ocrDetail = safeQuote((argv.ocrDetail != null && argv.ocrDetail != 'undefined') ? argv.ocrDetail : 'none'); // none, line, word -const ocrPSM = parseInt((argv.ocrPSM != null && argv.ocrPSM != 'undefined') ? argv.ocrPSM : 7); -const ocrOEM = parseInt((argv.ocrOEM != null && argv.ocrOEM != 'undefined') ? argv.ocrOEM : 3); +const imagePath = str(argv.imagePath, 'Screen'); +const imageSimilarity = parseFloat(str(argv.imageSimilarity, String(process.env.imageSimilarity || 0.8))); +const maxSim = parseFloat(str(argv.maxSim, '1')); +const textHint = str(argv.textHint, ''); +const imageAction = str(argv.imageAction, 'none'); +const imageWaitTime = parseInt(str(argv.imageWaitTime, String(process.env.imageWaitTime || 1))); +const imageMaxCount = parseInt(str(argv.imageMaxCount, '1')); +const flashSecs = (argv.flash != null && argv.flash !== 'undefined') ? parseFloat(argv.flash) : 1.0; + +// OCR-specific arguments (all opt-in). ocrPath === null means "image matching mode". +const ocrPath = (argv.ocrPath != null && argv.ocrPath !== 'undefined') ? String(argv.ocrPath) : null; +const ocrSimilarity = parseFloat(str(argv.ocrSimilarity, '0.8')); +const ocrMaxSim = parseFloat(str(argv.ocrMaxSim, '1.0')); +const ocrWaitTime = parseInt(str(argv.ocrWaitTime, '1000')); +const ocrMaxCount = parseInt(str(argv.ocrMaxCount, '1')); +const ocrAction = str(argv.ocrAction, 'none'); +const ocrDetail = str(argv.ocrDetail, 'none'); // none | line | word +const ocrPSM = parseInt(str(argv.ocrPSM, '7')); +const ocrOEM = parseInt(str(argv.ocrOEM, '3')); // default output const notFoundStatus = {status: 'notFound'}; -// Sikuli Property: App, Button, Mouse, OCR, Pattern, Region, Settings, Screen - // Flash the found region. Under Oculix/java-bridge Region.highlight() is // fire-and-forget (paints on a background thread and returns immediately), so if // this process calls process.exit() right after, the red box is torn down before -// it ever paints (the old synchronous silulix highlight did not have this -// problem). Flash for a visible duration and then hold the process so the box -// actually renders and is observable (by eye / VNC / screen recording). +// it ever paints. Flash for a visible duration and hold the process so the box +// actually renders. const flashOnMatch = (region) => { region.highlight(); Thread.sleep(Math.round(flashSecs * 1000)); }; +// Map a Region/rectangle-like object to the {location,dimension,center} triple. +const fillRectangleInfo = (rectItem) => { + const location = {x: rectItem.x, y: rectItem.y}; + const dimension = {width: rectItem.w, height: rectItem.h}; + const center = {x: rectItem.x + Math.round(rectItem.w / 2), y: rectItem.y + Math.round(rectItem.h / 2)}; + return [location, dimension, center]; +}; + +// Perform a mouse action on a rectangle-like object. Returns true when the action +// actually clicks (so the caller knows whether to record `clicked`). +const performAction = (rect, action) => { + if (!action || action === 'none' || action === 'null') return false; + const clickRegion = new Region(rect.x, rect.y, rect.w, rect.h); + clickRegion.mouseUpSync(); + let didClick = false; + switch (action) { + case 'single': + case 'click': + if (myDISPLAY.split(':')[1] > 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + didClick = true; + break; + case 'hoverClick': + clickRegion.hoverSync(); + if (myDISPLAY.split(':')[1] > 9) { + clickRegion.doubleClick(); + } else { + clickRegion.click(); + } + didClick = true; + break; + case 'double': + case 'doubleClick': + clickRegion.doubleClick(); + didClick = true; + break; + case 'right': + case 'rightClick': + clickRegion.rightClick(); + didClick = true; + break; + case 'hover': + clickRegion.hoverSync(); + break; + } + clickRegion.mouseUpSync(); + return didClick; +}; + // --------------------------------------------------------------------------- -// Core findImage function (image matching) +// Image-matching mode (the frozen original behaviour) // --------------------------------------------------------------------------- const findImage = (imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount) => { - // all input vars should be parsed or quoted - const myImagePath = safeQuote(imagePath); + const myImagePath = imagePath; const myImageName = myImagePath.substring(myImagePath.lastIndexOf('/') + 1); const myImageSimilarity = parseFloat(imageSimilarity); const myMaxSim = parseFloat(maxSim); - const myTextHint = safeQuote(textHint); + const myTextHint = textHint; const myImageWaitTime = parseInt(imageWaitTime); - const myImageAction = safeQuote(imageAction); + const myImageAction = imageAction; const myImageMaxCount = parseInt(imageMaxCount || 1); const findRegion = new Screen(); findRegion.setAutoWaitTimeout(myImageWaitTime); + let returnArray = []; try { var oneTarget; var returnItem = {name: myImageName, score: null, text: null, location: null, dimension: null, center: null, clicked: null}; - var returnArray = []; - const fillRectangleInfo = (rectItem) => { - const location = {x: rectItem.x, y: rectItem.y}; - const dimension = {width: rectItem.w, height: rectItem.h}; - const center = {x: rectItem.x + Math.round(rectItem.w / 2), y: rectItem.y + Math.round(rectItem.h / 2)}; - return [location, dimension, center]; - } if (myImagePath.includes('Screen')) { const screenMargin = myImagePath.includes('-') ? parseInt(myImagePath.split('-')[1]) : 1; oneTarget = new Region(findRegion.getBoundsSync()).growSync(-screenMargin); @@ -139,44 +181,11 @@ const findImage = (imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, } if (returnArray.length == 0) { returnArray.push(notFoundStatus); - } else { - // process myImageAction if any - if (myImageAction && myImageAction != 'none' && myImageAction != 'null') { - for (let i=0; i 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - break; - case 'hoverClick': - clickRegion.hoverSync(); - if (myDISPLAY.split(':')[1] > 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - break; - case 'double': - case 'doubleClick': - clickRegion.doubleClick(); - break; - case 'right': - case 'rightClick': - clickRegion.rightClick(); - break; - case 'hover': - clickRegion.hoverSync(); - break; - } - clickRegion.mouseUpSync(); - returnArray[i].clicked = returnArray[i].center; - } + } else if (myImageAction && myImageAction != 'none' && myImageAction != 'null') { + for (let i=0; i { - // OCR mode: ignore imagePath/imageAction/etc; use ocrPath as the text to find const waitStart = Date.now(); const results = []; - // Set OCR engine options via Settings (affects Region.textSync etc.) - Settings.OcrPSM = ocrPSM; - Settings.OcrOEM = ocrOEM; + try { + Settings.OcrPSM = ocrPSM; + Settings.OcrOEM = ocrOEM; + } catch (e) { /* older Oculix builds do not expose these knobs */ } while (Date.now() - waitStart < ocrWaitTime) { - // Capture the whole screen - const findRegion = new Screen(); - const img = findRegion.capture(); // returns BufferedImage - // Use Region.findText to get a Match with bounding box and confidence - const region = new Region(0, 0, img.getWidth(), img.getHeight()); - const similarity = ocrSimilarity; // 0-1 - const match = region.findText(ocrPath, similarity); - if (match !== null) { - // We have a match – build result - const rect = match; // Match implements getX, getY, getW, getH - const conf = match.getScore(); // 0-1 float - const text = match.getText(); // the matched string + let rect = null, text = ocrPath, conf = null; - // Build ocrDetails per requested granularity - const details = []; - if (ocrDetail === 'line') { - // Treat the whole match as one line - details.push({text, x:rect.getX(), y:rect.getY(), width:rect.getW(), height:rect.getH(), confidence:conf}); - } else if (ocrDetail === 'word') { - // Word-level detail would require iterating over OCREngine.getIterator(). - // For MVP we fall back to line-level; a future enhancement could implement true word boxes. - details.push({text, x:rect.getX(), y:rect.getY(), width:rect.getW(), height:rect.getH(), confidence:conf}); - } - // If none, details stays empty. + try { + const findRegion = new Screen(); + const screenRegion = new Region(findRegion.getBoundsSync()); + const region = new Region(0, 0, screenRegion.w, screenRegion.h); - // Base result fields - const result = { - name: text, - score: conf, - text: [text], // single line for simplicity - location: {x: rect.getX(), y: rect.getY(), width: rect.getW(), height: rect.getH()}, - dimension: {width: rect.getW(), height: rect.getH()}, - center: {x: rect.getX() + Math.round(rect.getW()/2), y: rect.getY() + Math.round(rect.getH()/2)}, - clicked: null, - ...(ocrDetail !== 'none' ? {ocrDetails: details} : {}) - }; + let tight = null; + try { + tight = region.findText(ocrPath, ocrSimilarity); + } catch (e) { + tight = null; + } - // Apply requested action - if (ocrAction && ocrAction !== 'none') { - // Reuse the same clickRegion logic as image matches - const clickRegion = new Region(result.location.x, result.location.y, result.dimension.width, result.dimension.height); - clickRegion.mouseUpSync(); - switch (ocrAction) { - case 'single': - case 'click': - if (myDISPLAY.split(':')[1] > 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - break; - case 'hoverClick': - clickRegion.hoverSync(); - if (myDISPLAY.split(':')[1] > 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - break; - case 'double': - case 'doubleClick': - clickRegion.doubleClick(); - break; - case 'right': - case 'rightClick': - clickRegion.rightClick(); - break; - case 'hover': - clickRegion.hoverSync(); - break; + if (tight) { + rect = {x: tight.getX(), y: tight.getY(), w: tight.getW(), h: tight.getH()}; + conf = tight.getScore(); + text = tight.getText(); + } else { + const needle = String(ocrPath).toLowerCase(); + const lines = region.textSync().split('\n'); + if (!lines.some((l) => String(l).toLowerCase().includes(needle))) { + Thread.sleep(50); + continue; } - clickRegion.mouseUpSync(); - result.clicked = result.center; + rect = {x: screenRegion.x, y: screenRegion.y, w: screenRegion.w, h: screenRegion.h}; + conf = 1.0; } + } catch (e) { + const msg = (e && typeof e.getMessageSync === 'function') ? e.getMessageSync() : (e && e.message ? e.message : String(e)); + console.log('findTargetImage ERROR:', msg); + return JSON.stringify([notFoundStatus]); + } - results.push(result); - // If we have enough matches, break - if (results.length >= ocrMaxCount) break; + const details = []; + if (ocrDetail === 'line' || ocrDetail === 'word') { + details.push({text: text, x: rect.x, y: rect.y, width: rect.w, height: rect.h, confidence: conf}); } - // Optional: small sleep before retry to avoid busy loop - java.lang.Thread.sleep(50); // Note: using java.lang.Thread directly; if this fails, we can use Thread.sleep as well. + + const result = { + name: text, + score: conf, + text: [text], + location: {x: rect.x, y: rect.y, width: rect.w, height: rect.h}, + dimension: {width: rect.w, height: rect.h}, + center: {x: rect.x + Math.round(rect.w/2), y: rect.y + Math.round(rect.h/2)}, + clicked: null, + ...(ocrDetail !== 'none' ? {ocrDetails: details} : {}) + }; + + if (performAction(rect, ocrAction)) { + result.clicked = result.center; + } + + results.push(result); + if (results.length >= ocrMaxCount) break; + Thread.sleep(50); } if (results.length === 0) { - return [notFoundStatus]; + return JSON.stringify([notFoundStatus]); } - return results; + return JSON.stringify(results); }; // --------------------------------------------------------------------------- @@ -295,13 +285,11 @@ const findImageOcr = (ocrPath, ocrSimilarity, ocrMaxSim, ocrWaitTime, ocrMaxCoun // --------------------------------------------------------------------------- let target_result; if (ocrPath !== null) { - // OCR mode target_result = findImageOcr(ocrPath, ocrSimilarity, ocrMaxSim, ocrWaitTime, ocrMaxCount, ocrAction, ocrDetail, ocrPSM, ocrOEM); } else { - // Image matching mode (original behavior) target_result = findImage(imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount); } // display result to stdout console.log(`target_result: ${target_result}`); -process.exit(); \ No newline at end of file +process.exit(); diff --git a/seam/src/findTargetImage.js.bak2 b/seam/src/findTargetImage.js.bak2 deleted file mode 100755 index 05a816c9..00000000 --- a/seam/src/findTargetImage.js.bak2 +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env node - -// java-bridge replaces node-java (JNI); set JVM options up front so they apply -// before the JVM is started by the first class import/call below. -const java = require('java-bridge'); -java.ensureJvm({ opts: ['-Xms128m', '-Xmx512m'] }); - -const path = require('path'); -const safeQuote = require('./safequote'); -const minimist = require('minimist'); - -// all external env vars should be parsed or quoted to const -process.env.imageSimilarity = parseFloat(process.env.imageSimilarity) || 0.8; -process.env.imageWaitTime = parseInt(process.env.imageWaitTime) || 1; -process.env.TESSDATA_PREFIX = safeQuote(process.env.TESSDATA_PREFIX) || '/usr/share/tesseract-ocr/4.00/tessdata'; -process.env.OMP_THREAD_LIMIT = parseInt(process.env.OMP_THREAD_LIMIT) || 1; -const myDISPLAY = ':' + (process.env.DISPLAY ? parseInt(process.env.DISPLAY.split(':')[1]) : 1); - -// needed for Tesseract-OCR property -process.env.LC_ALL = 'C'; -process.env.LC_CTYPE = 'C'; - -// Determine Oculix JAR location: seam/lib/oculixapi--linux.jar -// Allow overriding version via OCULIX_VER env (set at build time) -const OCULIX_VER = process.env.OCULIX_VER || '4.0.0'; -const jarPath = path.join(__dirname, '..', 'lib', `oculixapi-${OCULIX_VER}-linux.jar`); -java.classpath.append(jarPath); - -// import classes for export -const { - App, - Button, - ImagePath, - Mouse, - OCR, - Pattern, - Region, - Settings, - Screen -} = java.import( - 'org.sikuli.script.App', - 'org.sikuli.script.Button', - 'org.sikuli.script.ImagePath', - 'org.sikuli.script.Mouse', - 'org.sikuli.script.OCR', - 'org.sikuli.script.Pattern', - 'org.sikuli.script.Region', - 'org.sikuli.basics.Settings', - 'org.sikuli.script.Screen' -); - -// all args should be parsed or quoted to const -const argv = minimist(process.argv.slice(2)); -const imagePath = safeQuote((argv.imagePath != null && argv.imagePath != 'undefined') ? argv.imagePath : 'Screen'); -const imageSimilarity = parseFloat((argv.imageSimilarity != null && argv.imageSimilarity != 'undefined') ? argv.imageSimilarity : process.env.imageSimilarity || 0.8); -const maxSim = parseFloat((argv.maxSim != null && argv.maxSim != 'undefined') ? argv.maxSim : 1); -const textHint = safeQuote((argv.textHint != null && argv.textHint != 'undefined') ? argv.textHint : ''); -const imageAction = safeQuote((argv.imageAction != null && argv.imageAction != 'undefined') ? argv.imageAction : 'none'); -const imageWaitTime = parseInt((argv.imageWaitTime != null && argv.imageWaitTime != 'undefined') ? argv.imageWaitTime : process.env.imageWaitTime || 1); -const imageMaxCount = parseInt((argv.imageMaxCount != null && argv.imageMaxCount != 'undefined') ? argv.imageMaxCount : 1); -const flashSecs = (argv.flash != null && argv.flash != 'undefined') ? parseFloat(argv.flash) : 1.0; - -// OCR-specific arguments (all opt-in) -const ocrPath = safeQuote((argv.ocrPath != null && argv.ocrPath != 'undefined') ? argv.ocrPath : null); -const ocrSimilarity = parseFloat((argv.ocrSimilarity != null && argv.ocrSimilarity != 'undefined') ? argv.ocrSimilarity : 0.8); -const ocrMaxSim = parseFloat((argv.ocrMaxSim != null && argv.ocrMaxSim != 'undefined') ? argv.ocrMaxSim : 1.0); -const ocrWaitTime = parseInt((argv.ocrWaitTime != null && argv.ocrWaitTime != 'undefined') ? argv.ocrWaitTime : 1000); -const ocrMaxCount = parseInt((argv.ocrMaxCount != null && argv.ocrMaxCount != 'undefined') ? argv.ocrMaxCount : 1); -const ocrAction = safeQuote((argv.ocrAction != null && argv.ocrAction != 'undefined') ? argv.ocrAction : 'none'); -const ocrDetail = safeQuote((argv.ocrDetail != null && argv.ocrDetail != 'undefined') ? argv.ocrDetail : 'none'); // none, line, word -const ocrPSM = parseInt((argv.ocrPSM != null && argv.ocrPSM != 'undefined') ? argv.ocrPSM : 7); -const ocrOEM = parseInt((argv.ocrOEM != null && argv.ocrOEM != 'undefined') ? argv.ocrOEM : 3); - -// default output -const notFoundStatus = {status: 'notFound'}; - -// Sikuli Property already imported above: App, Button, Mouse, OCR, Pattern, Region, Settings, Screen - -// Flash the found region. Under Oculix/java-bridge Region.highlight() is -// fire-and-forget (paints on a background thread and returns immediately), so if -// this process calls process.exit() right after, the red box is torn down before -// it ever paints (the old synchronous silulix highlight did not have this -// problem). Flash for a visible duration and then hold the process so the box -// actually renders and is observable (by eye / VNC / screen recording). -const flashOnMatch = (region) => { - region.highlight(); - java.lang.Thread.sleep(Math.round(flashSecs * 1000)); -}; - -// --------------------------------------------------------------------------- -// Core findImage function -// --------------------------------------------------------------------------- -const findImage = (imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount) => { - // all input vars should be parsed or quoted - const myImagePath = safeQuote(imagePath); - const myImageName = myImagePath.substring(myImagePath.lastIndexOf('/') + 1); - const myImageSimilarity = parseFloat(imageSimilarity); - const myMaxSim = parseFloat(maxSim); - const myTextHint = safeQuote(textHint); - const myImageWaitTime = parseInt(imageWaitTime); - const myImageAction = safeQuote(imageAction); - const myImageMaxCount = parseInt(imageMaxCount || 1); - - const findRegion = new Screen(); - findRegion.setAutoWaitTimeout(myImageWaitTime); - - try { - var oneTarget; - var returnItem = {name: myImageName, score: null, text: null, location: null, dimension: null, center: null, clicked: null}; - var returnArray = []; - const fillRectangleInfo = (rectItem) => { - const location = {x: rectItem.x, y: rectItem.y}; - const dimension = {width: rectItem.w, height: rectItem.h}; - const center = {x: rectItem.x + Math.round(rectItem.w / 2), y: rectItem.y + Math.round(rectItem.h / 2)}; - return [location, dimension, center]; - } - if (myImagePath.includes('Screen')) { - const screenMargin = myImagePath.includes('-') ? parseInt(myImagePath.split('-')[1]) : 1; - oneTarget = new Region(findRegion.getBoundsSync()).growSync(-screenMargin); - returnItem.text = oneTarget.textSync().split('\n'); - [returnItem.location, returnItem.dimension, returnItem.center] = fillRectangleInfo(oneTarget); - flashOnMatch(oneTarget); - returnArray.push(returnItem); - } else { - const oneSample = (new Pattern(myImagePath)).similarSync(myImageSimilarity); - const findTargets = findRegion.findAllSync(oneSample); - const myRegex = new RegExp(myTextHint, 'i'); - var matchCount = 0; - while (matchCount < myImageMaxCount && findTargets.hasNextSync()) { - const oneMatch = findTargets.nextSync(); - returnItem.score = Math.floor(oneMatch.getScoreSync()*1000000)/1000000; - [returnItem.location, returnItem.dimension, returnItem.center] = fillRectangleInfo(oneMatch); - oneTarget = new Region(oneMatch); - returnItem.text = oneTarget.textSync().split('\n'); - if (returnItem.score >= myImageSimilarity && returnItem.score <= myMaxSim && returnItem.text.join('\n').match(myRegex)) { - matchCount += 1; - flashOnMatch(oneTarget); - returnArray.push(returnItem); - } - } - } - if (returnArray.length == 0) { - returnArray.push(notFoundStatus); - } else { - // process myImageAction if any - if (myImageAction && myImageAction != 'none' && myImageAction != 'null') { - for (let i=0; i 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - break; - case 'hoverClick': - clickRegion.hoverSync(); - if (myDISPLAY.split(':')[1] > 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - break; - case 'double': - case 'doubleClick': - clickRegion.doubleClick(); - break; - case 'right': - case 'rightClick': - clickRegion.rightClick(); - break; - case 'hover': - clickRegion.hoverSync(); - break; - } - clickRegion.mouseUpSync(); - returnArray[i].clicked = returnArray[i].center; - } - } - } - } catch(e) { - const msg = (e && typeof e.getMessageSync === 'function') ? e.getMessageSync() : (e && e.message ? e.message : String(e)); - console.log('findTargetImage ERROR:', msg); - returnArray.push(notFoundStatus); - } finally { - return JSON.stringify(returnArray); - } -}; - -// --------------------------------------------------------------------------- -// OCR mode function (new) -// --------------------------------------------------------------------------- -const findImageOcr = (ocrPath, ocrSimilarity, ocrMaxSim, ocrWaitTime, ocrMaxCount, ocrAction, ocrDetail, ocrPSM, ocrOEM) => { - // OCR mode: ignore imagePath/imageAction/etc; use ocrPath as the text to find - const waitStart = Date.now(); - const results = []; - - // Set OCR engine options via Settings (affects Region.textSync etc.) - Settings.OcrPSM = ocrPSM; - Settings.OcrOEM = ocrOEM; - - while (Date.now() - waitStart < ocrWaitTime) { - // Capture the whole screen (or a subregion if imagePath also given? We'll ignore imagePath for pure OCR mode; - // but if user also supplied --imagePath we could treat it as a region constraint. - // For simplicity, we follow the existing behavior: if imagePath is not 'Screen', we treat it as a region constraint. - let findRegion; - if (imagePath && !imagePath.includes('Screen')) { - // imagePath is a file; we cannot use it as a region directly. - // Fallback to full screen. - findRegion = new Screen(); - } else { - findRegion = new Screen(); - } - const img = findRegion.capture(); // returns BufferedImage - // We'll use Region OCR via textSync with bounding boxes? Unfortunately Region.textSync() returns only text. - // To get boxes we need to use the OCREngine API via java. - // Let's use the OCREngine directly: - const ocrEngine = new OCR(); - // The OCREngine can recognize an image and return a ResultIterator. - // However, the simplest way that works with the bundled Oculix is to use Region.findText which returns a Match with bounds. - // We'll attempt to find the text using Region.findText with similarity. - const region = new Region(0, 0, img.getWidth(), img.getHeight()); - // Use the OCR engine's findText method? Actually Region has findText(String pattern, double similarity). - // We'll use that. - const similarity = ocrSimilarity; // 0-1 - const match = region.findText(ocrPath, similarity); - if (match !== null) { - // We have a match – build result - const rect = match; // Match implements getX, getY, getW, getH - const conf = match.getScore(); // 0-1 float - const text = match.getText(); // the matched string - - // Build ocrDetails per requested granularity - const details = []; - if (ocrDetail === 'line') { - // For line-level we could split by \n and approximate each line's box. - // Simpler: treat the whole match as one line. - details.push({text, x:rect.getX(), y:rect.getY(), width:rect.getW(), height:rect.getH(), confidence:conf}); - } else if (ocrDetail === 'word') { - // Word-level detail would require iterating over OCREngine.getIterator(). - // For MVP we fall back to line-level; a future enhancement could implement true word boxes. - details.push({text, x:rect.getX(), y:rect.getY(), width:rect.getW(), height:rect.getH(), confidence:conf}); - } - // If none, details stays empty. - - // Base result fields - const result = { - name: text, - score: conf, - text: [text], // single line for simplicity - location: {x: rect.getX(), y: rect.getY(), width: rect.getW(), height: rect.getH()}, - dimension: {width: rect.getW(), height: rect.getH()}, - center: {x: rect.getX() + Math.round(rect.getW()/2), y: rect.getY() + Math.round(rect.getH()/2)}, - clicked: null, - ...(ocrDetail !== 'none' ? {ocrDetails: details} : {}) - }; - - // Apply requested action - if (ocrAction && ocrAction !== 'none') { - // Reuse the same clickRegion logic as image matches - const clickRegion = new Region(result.location.x, result.location.y, result.dimension.width, result.dimension.height); - clickRegion.mouseUpSync(); - switch (ocrAction) { - case 'single': - case 'click': - if (myDISPLAY.split(':')[1] > 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - break; - case 'hoverClick': - clickRegion.hoverSync(); - if (myDISPLAY.split(':')[1] > 9) { - clickRegion.doubleClick(); - } else { - clickRegion.click(); - } - break; - case 'double': - case 'doubleClick': - clickRegion.doubleClick(); - break; - case 'right': - case 'rightClick': - clickRegion.rightClick(); - break; - case 'hover': - clickRegion.hoverSync(); - break; - } - clickRegion.mouseUpSync(); - result.clicked = result.center; - } - - results.push(result); - // If we have enough matches, break - if (results.length >= ocrMaxCount) break; - } - // Optional: small sleep before retry to avoid busy loop - java.lang.Thread.sleep(50); - } - - if (results.length === 0) { - return [notFoundStatus]; - } - return results; -}; - -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- -let target_result; -if (ocrPath !== null) { - // OCR mode - target_result = findImageOcr(ocrPath, ocrSimilarity, ocrMaxSim, ocrWaitTime, ocrMaxCount, ocrAction, ocrDetail, ocrPSM, ocrOEM); -} else { - // Image matching mode (original behavior) - target_result = findImage(imagePath, imageSimilarity, maxSim, textHint, imageWaitTime, imageAction, imageMaxCount); -} - -// display result to stdout -console.log(`target_result: ${target_result}`); -process.exit(); \ No newline at end of file diff --git a/seam/src/safequote.js b/seam/src/safequote.js deleted file mode 100644 index 5095a18a..00000000 --- a/seam/src/safequote.js +++ /dev/null @@ -1,5 +0,0 @@ -// prepare for safeQuote -const quote = require('shell-quote').quote; -const parse = require('shell-quote').parse; -const safequote = (str) => { return (str) ? quote(parse(str.replace(/#/g, ''))) : undefined }; -module.exports = safequote; \ No newline at end of file diff --git a/test-projects/autobdd-base-test/base-test/run.sh b/test-projects/autobdd-base-test/base-test/run.sh index e7b6b189..dfdb8fdb 100755 --- a/test-projects/autobdd-base-test/base-test/run.sh +++ b/test-projects/autobdd-base-test/base-test/run.sh @@ -107,25 +107,25 @@ sleep 2 # OCR detection (no action) JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrDetail=word)" check_eq "ocr word count" "$(jq1 "$JSON" 'length')" "1" -check_eq "ocr word text" "$(jq1 "$JSON" '.[0].ocrDetails[0].text')" '"AUTOTEST OCR"' +check_eq "ocr word text" "$(jq1 "$JSON" '.[0].ocrDetails[0].text')" "AUTOTEST OCR" check_has "ocr word has bbox" "$(jq1 "$JSON" '.[0].ocrDetails[0] | has("x") and has("y") and has("width") and has("height")')" "true" # OCR click action JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=click --ocrDetail=word)" -check_eq "ocr clicked point type" "$(jq1 "$JSON" '.[0].clicked|type')" '"object"' +check_eq "ocr clicked point type" "$(jq1 "$JSON" '.[0].clicked|type')" "object" check_eq "ocr clicked x matches center" "$(jq1 "$JSON" '.[0].clicked.x')" "$(jq1 "$JSON" '.[0].ocrDetails[0].x + .[0].ocrDetails[0].width/2')" check_eq "ocr clicked y matches center" "$(jq1 "$JSON" '.[0].clicked.y')" "$(jq1 "$JSON" '.[0].ocrDetails[0].y + .[0].ocrDetails[0].height/2')" # OCR doubleClick action JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=doubleClick --ocrDetail=word)" -check_eq "ocr doubleClicked" "$(jq1 "$JSON" '.[0].clicked|type')" '"object"' +check_eq "ocr doubleClicked" "$(jq1 "$JSON" '.[0].clicked|type')" "object" # OCR rightClick action JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=rightClick --ocrDetail=word)" -check_eq "ocr rightClicked" "$(jq1 "$JSON" '.[0].clicked|type')" '"object"' +check_eq "ocr rightClicked" "$(jq1 "$JSON" '.[0].clicked|type')" "object" # OCR hover action (no click) JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=hover --ocrDetail=word)" check_eq "ocr hovered (no clicked)" "$(jq1 "$JSON" '.[0].clicked')" "null" # OCR hoverClick action JSON="$(seam --ocrPath="AUTOTEST OCR" --ocrAction=hoverClick --ocrDetail=word)" -check_eq "ocr hoverClicked" "$(jq1 "$JSON" '.[0].clicked|type')" '"object"' +check_eq "ocr hoverClicked" "$(jq1 "$JSON" '.[0].clicked|type')" "object" # --------------------------------------------------------------------------- section "L1 — keyboard/mouse substrate" # ---------------------------------------------------------------------------