Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@

## Unreleased

- Add `--platform` to build smaller npm tarballs for one supported operating system and architecture while retaining the default universal package.
- Preserve every scanner report when sanitized target, profile, or custom scanner names collide with each other or generated numeric suffixes.
- Fix unbounded memory use when loading benchmark `--ids` from files or HTTP, including whitespace-padded IDs; document selection limits and preserve full-set JSONL support. Thanks @SebTardif (#47).
30 changes: 30 additions & 0 deletions npm/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Building npm packages

The default build produces the universal `@openclaw/clawscan` package with all
six supported binaries:

```sh
node scripts/build-npm-package.mjs --version v0.0.0 --pack --smoke
```

Use `--platform` to build a smaller tarball containing one binary. For example:

```sh
node scripts/build-npm-package.mjs --version v0.0.0 --platform linux-x64 --out dist/npm-linux-x64 --pack
npm install --global ./dist/npm-linux-x64/openclaw-clawscan-linux-x64-0.0.0.tgz
clawscan --version
```

Supported values are `darwin-x64`, `darwin-arm64`, `linux-x64`, `linux-arm64`,
`win32-x64`, and `win32-arm64`. The package name is
`@openclaw/clawscan-<platform>`, and its `os` and `cpu` fields restrict installation
to that platform. Each package keeps the `clawscan` command and the
`./resolve-binary` export, with its binary under `binaries/<platform>/`.

Use a distinct `--out` directory to retain multiple builds; each build replaces
its output directory. Add `--smoke` when building for the current host to install
the tarball in a temporary prefix and run `--version` and a static scan.

The platform-specific tarballs are local build artifacts. Registry publication
still uses the universal package; publishing the additional names requires their
registry bootstrap, trusted-publisher setup, and release-workflow integration.
42 changes: 35 additions & 7 deletions scripts/build-npm-package.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@ export function binaryNameForTarget(target) {
return target.goos === "windows" ? "clawscan.exe" : "clawscan";
}

export function packageTargetForPlatform(platform) {
const target = packageTargets.find((entry) => platformKeyForTarget(entry) === platform);
if (!target) {
const supported = packageTargets.map(platformKeyForTarget).join(", ");
throw new Error(
`Unsupported npm package platform: ${platform ?? "<missing>"}. Expected ${supported}.`,
);
}
return target;
}

function run(command, args, options = {}) {
const result = spawnSync(command, args, {
cwd: options.cwd ?? repoRoot,
Expand Down Expand Up @@ -91,6 +102,10 @@ function parseArgs(argv) {
options.outDir = resolve(argv[++index]);
continue;
}
if (arg === "--platform") {
options.target = packageTargetForPlatform(argv[++index]);
continue;
}
if (arg === "--pack") {
options.pack = true;
continue;
Expand Down Expand Up @@ -132,10 +147,17 @@ async function stagePackage(options) {
const packageJsonPath = join(packageOut, "package.json");
const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
packageJson.version = packageVersion;
if (options.target) {
const platformKey = platformKeyForTarget(options.target);
const [platform, arch] = platformKey.split("-");
packageJson.name = `${packageJson.name}-${platformKey}`;
packageJson.os = [platform];
packageJson.cpu = [arch];
}
await writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`);

const ldflags = `-s -w -X main.version=${binaryVersion} -X main.commit=${releaseCommit} -X main.date=${buildDate}`;
for (const target of packageTargets) {
for (const target of options.target ? [options.target] : packageTargets) {
const binaryDir = join(packageOut, "binaries", platformKeyForTarget(target));
await mkdir(binaryDir, { recursive: true });
run(
Expand Down Expand Up @@ -164,28 +186,34 @@ async function stagePackage(options) {
await writeFile(join(options.outDir, "release-sha.txt"), `${releaseSha}\n`);
await writeFile(join(options.outDir, "package-version.txt"), `${packageVersion}\n`);

return { binaryVersion, packageOut, packageVersion, releaseSha };
return {
binaryVersion,
packageName: packageJson.name,
packageOut,
packageVersion,
releaseSha,
};
}

export function parsePackFilename(output) {
export function parsePackFilename(output, packageName = "@openclaw/clawscan") {
const parsed = JSON.parse(output);
// npm 12 keys results by package name; older supported npm versions use an array.
const packed = Array.isArray(parsed) ? parsed[0] : parsed?.["@openclaw/clawscan"];
const packed = Array.isArray(parsed) ? parsed[0] : parsed?.[packageName];
if (typeof packed?.filename !== "string" || !packed.filename) {
throw new Error("npm pack did not return a tarball filename.");
}
return packed.filename;
}

async function packPackage(options, packageOut) {
async function packPackage(options, packageOut, packageName) {
const result = run(
"npm",
["pack", "--json", "--ignore-scripts", "--pack-destination", options.outDir],
{
cwd: packageOut,
},
);
return resolve(options.outDir, parsePackFilename(result.stdout));
return resolve(options.outDir, parsePackFilename(result.stdout, packageName));
}

async function smokePackage(tarballPath, binaryVersion) {
Expand Down Expand Up @@ -215,7 +243,7 @@ export async function main(argv = process.argv.slice(2)) {
const staged = await stagePackage(options);
let tarballPath = "";
if (options.pack) {
tarballPath = await packPackage(options, staged.packageOut);
tarballPath = await packPackage(options, staged.packageOut, staged.packageName);
}
if (options.smoke) {
await smokePackage(tarballPath, staged.binaryVersion);
Expand Down
39 changes: 39 additions & 0 deletions scripts/build-npm-package.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@ import { readFileSync } from "node:fs";
import { describe, it } from "node:test";
import {
binaryNameForTarget,
main,
normalizeBuildDate,
normalizePackageVersion,
npmDistTagForVersion,
packageTargets,
packageTargetForPlatform,
parsePackFilename,
platformKeyForTarget,
} from "./build-npm-package.mjs";
Expand All @@ -22,6 +24,21 @@ describe("npm pack output", () => {
assert.equal(parsePackFilename(JSON.stringify([{ filename }])), filename);
});

it("reads the selected platform package from npm 11 and npm 12 results", () => {
const name = "@openclaw/clawscan-win32-arm64";
const platformFilename = "openclaw-clawscan-win32-arm64-1.2.3.tgz";
for (const result of [
{ [name]: { filename: platformFilename } },
[{ filename: platformFilename }],
]) {
assert.equal(parsePackFilename(JSON.stringify(result), name), platformFilename);
}
assert.throws(
() => parsePackFilename(JSON.stringify({ "@openclaw/clawscan": { filename } }), name),
/did not return a tarball filename/,
);
});

it("rejects missing or invalid tarball filenames", () => {
for (const result of [null, {}, [], [{ filename: "" }], { "@openclaw/clawscan": { filename: 42 } }]) {
assert.throws(() => parsePackFilename(JSON.stringify(result)), /did not return a tarball filename/);
Expand Down Expand Up @@ -67,6 +84,28 @@ describe("normalizeBuildDate", () => {
});

describe("package target mapping", () => {
it("selects one Go target for each supported npm platform", () => {
for (const [platform, target] of [
["darwin-x64", { goos: "darwin", goarch: "amd64" }],
["darwin-arm64", { goos: "darwin", goarch: "arm64" }],
["linux-x64", { goos: "linux", goarch: "amd64" }],
["linux-arm64", { goos: "linux", goarch: "arm64" }],
["win32-x64", { goos: "windows", goarch: "amd64" }],
["win32-arm64", { goos: "windows", goarch: "arm64" }],
]) {
assert.deepEqual(packageTargetForPlatform(platform), target);
}
});

it("rejects missing or unsupported --platform values before staging output", async () => {
for (const args of [[], ["freebsd-x64"], ["linux-ia32"], ["windows-x64"]]) {
await assert.rejects(
main(["--version", "v1.2.3", "--platform", ...args]),
/Unsupported npm package platform/,
);
}
});

it("maps Go release targets to npm binary directories", () => {
assert.deepEqual(
packageTargets.map((target) => [target.goos, target.goarch, platformKeyForTarget(target)]),
Expand Down
Loading