diff --git a/.github/workflows/skill-bundle-tests.yaml b/.github/workflows/skill-bundle-tests.yaml new file mode 100644 index 00000000..8786f6bc --- /dev/null +++ b/.github/workflows/skill-bundle-tests.yaml @@ -0,0 +1,24 @@ +name: Sandbox Skill bundle tests +on: + pull_request: + paths: + - 'base-images/frameworks/sandbox/v1/**' + - '.github/workflows/skill-bundle-tests.yaml' + push: + branches: [main] + paths: + - 'base-images/frameworks/sandbox/v1/**' + - '.github/workflows/skill-bundle-tests.yaml' +permissions: + contents: read +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - run: node --test base-images/frameworks/sandbox/v1/skill-bundle*.test.mjs + - name: Build and verify every Skill from the actual pinned repository + run: node base-images/frameworks/sandbox/v1/verify-pinned-source.mjs diff --git a/base-images/frameworks/sandbox/v1/Dockerfile b/base-images/frameworks/sandbox/v1/Dockerfile index 1ff83d33..bc32ec71 100644 --- a/base-images/frameworks/sandbox/v1/Dockerfile +++ b/base-images/frameworks/sandbox/v1/Dockerfile @@ -14,6 +14,18 @@ ARG NODE_IMAGE_VERSION=v0.0.1-alpha.1-${L10N_NORMALIZED} FROM ${REGISTRY}/${REPO_CODEX_GATEWAY}:${CODEX_GATEWAY_IMAGE_TAG} AS codex-gateway +FROM ${REGISTRY}/${REPO}/node.js-22:${NODE_IMAGE_VERSION} AS skills +ARG SEALOS_SKILLS_REPOSITORY=https://github.com/norberia/sealos-skills-next.git +ARG SEALOS_SKILLS_REVISION=bdd824cf2fd6c72896f8e201f32259cc8aed3f98 +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates && rm -rf /var/lib/apt/lists/* +COPY skill-bundle.mjs /build/skill-bundle.mjs +RUN git init /source && git -C /source remote add origin "$SEALOS_SKILLS_REPOSITORY" && \ + git -C /source fetch --depth 1 origin "$SEALOS_SKILLS_REVISION" && \ + git -C /source checkout --detach FETCH_HEAD && \ + test "$(git -C /source rev-parse HEAD)" = "$SEALOS_SKILLS_REVISION" && \ + /usr/bin/node /build/skill-bundle.mjs build /source/plugins/sealos /bundle "$SEALOS_SKILLS_REVISION" && \ + /usr/bin/node /build/skill-bundle.mjs verify /bundle + FROM ${REGISTRY}/${REPO}/node.js-22:${NODE_IMAGE_VERSION} ARG L10N LABEL org.opencontainers.image.authors="The Devbox Authors" @@ -52,6 +64,9 @@ ENV KANIKO_CONTEXT_S3_BASE=s3://kaniko-contexts/contexts ENV KANIKO_CONTEXT_POSIX_DIR=/home/${DEFAULT_DEVBOX_USER}/workspace/.versitygw-s3/kaniko-contexts/contexts COPY --from=codex-gateway /usr/local/bin/codex-gateway /usr/local/bin/codex-gateway +COPY --from=skills /bundle /opt/sealai/skill-bundle +COPY skill-bundle.mjs /opt/sealai/skill-bundle.mjs +COPY --chmod=755 prepare-skills /usr/local/bin/sealai-prepare-skills # Add build assets and execute them. COPY codex-gateway /tmp/codex-gateway-service @@ -64,6 +79,7 @@ RUN chmod +x \ /tmp/versitygw-service/run \ /tmp/versitygw-service/finish && \ /build.sh && \ + test -x /usr/bin/node && test -x /usr/bin/flock && test -x /usr/bin/timeout && \ rm -f /build.sh && \ rm -rf /tmp/codex-gateway-service /tmp/versitygw-service diff --git a/base-images/frameworks/sandbox/v1/README.md b/base-images/frameworks/sandbox/v1/README.md new file mode 100644 index 00000000..c6d2057c --- /dev/null +++ b/base-images/frameworks/sandbox/v1/README.md @@ -0,0 +1,39 @@ +# Offline managed Skills + +This image owns the Sealos Skill bundle; codex-gateway only supplies its existing binary. Network access to GitHub is needed during image build, never during Skill preparation. + +The `skills` build stage fetches `SEALOS_SKILLS_REPOSITORY` at the full `SEALOS_SKILLS_REVISION` commit. The default source is `https://github.com/norberia/sealos-skills-next.git`, pinned to `bdd824cf2fd6c72896f8e201f32259cc8aed3f98`, not a floating runtime branch. This preserves the repository selected by the former Brain `DEPLOY_SKILL_SOURCE` setting. Change these build arguments for a reviewed preview revision. Do not put private repository credentials in build arguments. + +The build reads every Skill in the repository's canonical `plugins/sealos/skills` tree, not the top-level `skills` symlink aliases. There is no Skill-name allowlist or fixed Skill count. The source tree determines the manifest; build/verify reject empty, malformed or incomplete bundles. Updating the pinned commit picks up the source's full Skill set. Repository overrides must provide the same plugin layout. + +The image contains: + +- `/opt/sealai/skill-bundle`: complete Skill resource trees and a schema-1 manifest (source commit, Skill names, file SHA-256). +- `/opt/sealai/skill-bundle.mjs`: build, verify and local preparation implementation using the existing Node runtime. +- `/usr/local/bin/sealai-prepare-skills`: fixed entry point for Brain, serialized with `flock`. Lock waiting and preparation share a 28-second budget (1-second kill grace), below Brain's 30-second RPC cap. Node, flock and timeout use absolute system paths; Node environment overrides are cleared. + +Run the entry point as the Devbox user after repository cloning; the workspace root must already exist. Chat has no repository clone and explicitly passes `--init-workspace`. Both modes prepare `/home/devbox/project/.agents/skills`, preserving unrelated Skills and `skills-lock.json`. Symlinks are rejected, and user files are type-checked without reading/hashing their contents. + +Each replacement uses an exclusive `mkdtemp` transaction under `.sealai-skill-transactions`, outside `.agents` and on the workspace filesystem. If killed between renames, the next invocation restores the backup before creating any live Skill directory. Stages are never promoted during recovery because they may be incomplete. After publication, cleanup is best-effort; ambiguous recovery state fails closed and preserves files for inspection. Legacy PID-named backups are restored only when there is one backup and no live tree. This helper is not a security boundary against concurrent malicious filesystem mutation by the workspace owner, nor a power-loss durability guarantee. + +The wrapper parses and validates success before printing schema, status, revision, bundle digest and Skill count as JSON. Empty/malformed output is failure. Failures use fixed codes, including preparation timeout, never file contents or raw exceptions. Missing or corrupt bundles fail closed. There is no npx/download fallback. + +## Verification + +From the repository root: + +```sh +node --test base-images/frameworks/sandbox/v1/skill-bundle*.test.mjs +node base-images/frameworks/sandbox/v1/verify-pinned-source.mjs +``` + +The existing Runtime Smoke workflow can run `tests/runtime-smoke/frameworks/sandbox/v1/smoke.sh` in the final sandbox image. For an offline acceptance check, run that script in a disposable image container with `--network none` as the `devbox` user. The real Devbox check is separate: clone a repository first, prepare as the actual user, and exercise Chat loadSkill/loadSkillResource and a GitHub Deployment Task. + +## Rollout + +1. Build and publish the sandbox/v1 base image with this bundle, then its runtime image if that is the deployment target. Use the existing runtime build pipeline; do not change the Gateway image for this feature. +2. Inspect the built image's manifest and run the offline smoke check. Retain the source revision and immutable image digest in release evidence. +3. Finish active/blocked Brain Deployment Tasks. Set Brain's `DEVBOX_RUNTIME_IMAGE` to the verified immutable sandbox image and remove nonempty `DEPLOY_SKILL_SOURCE` overrides while deploying the matching Brain change. +4. Confirm fresh Chat and GitHub runtimes prepare the same revision. Old Chat runtimes are not deleted; their existing lifecycle handles retention. + +Rollback restores the previous Brain/runtime image pair. Never compensate for a broken image by re-enabling a runtime internet installer. Image registry availability is still required to start a new Devbox; this change removes only Skill-install network dependencies. diff --git a/base-images/frameworks/sandbox/v1/build.sh b/base-images/frameworks/sandbox/v1/build.sh index eb076276..6741571e 100644 --- a/base-images/frameworks/sandbox/v1/build.sh +++ b/base-images/frameworks/sandbox/v1/build.sh @@ -51,7 +51,7 @@ esac apt-get update && \ apt-get install -y wget build-essential libncursesw5-dev libssl-dev bubblewrap \ - ripgrep \ + ripgrep util-linux \ libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/base-images/frameworks/sandbox/v1/prepare-skills b/base-images/frameworks/sandbox/v1/prepare-skills new file mode 100644 index 00000000..3d566010 --- /dev/null +++ b/base-images/frameworks/sandbox/v1/prepare-skills @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu +# Do not inherit repository-selected Node options or interpreter lookup paths. +unset NODE_OPTIONS NODE_PATH +operation=prepare +case "$#:$*" in + 0:) ;; + 1:--init-workspace) operation=prepare-chat ;; + *) printf '%s\n' '{"schema":1,"status":"failed","reason":"invalid_arguments"}' >&2; exit 1 ;; +esac +# One shared wait + work budget, leaving room under Brain's 30-second RPC cap. +if output=$(/usr/bin/timeout --kill-after=1 28 /usr/bin/flock /tmp/sealai-runtime-skills.lock /usr/bin/node /opt/sealai/skill-bundle.mjs "$operation" /opt/sealai/skill-bundle /home/devbox/project 2>/dev/null); then + if printf '%s' "$output" | /usr/bin/node /opt/sealai/skill-bundle.mjs validate-ready 2>/dev/null; then + exit 0 + fi + reason=invalid_ready +else + code=$? + reason=skill_bundle_unavailable + if [ "$code" = 124 ] || [ "$code" = 137 ]; then reason=skill_prepare_timeout; fi +fi +printf '{"schema":1,"status":"failed","reason":"%s"}\n' "$reason" >&2 +exit 1 diff --git a/base-images/frameworks/sandbox/v1/skill-bundle-regression.test.mjs b/base-images/frameworks/sandbox/v1/skill-bundle-regression.test.mjs new file mode 100644 index 00000000..78fcf02c --- /dev/null +++ b/base-images/frameworks/sandbox/v1/skill-bundle-regression.test.mjs @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, realpath, rename, rm, symlink, writeFile } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { buildBundle, prepareBundle, validateReady, verifyBundle } from './skill-bundle.mjs'; + +const revision = 'b'.repeat(40); +async function fixture(t, names = ['sealos-deploy']) { + const root = await mkdtemp(path.join(await realpath(os.tmpdir()), 'skills-regression-')); + t.after(() => rm(root, { recursive: true, force: true })); + const source = path.join(root, 'source'); + const bundle = path.join(root, 'bundle'); + const workspace = path.join(root, 'workspace'); + await mkdir(workspace); + for (const name of names) { + await mkdir(path.join(source, 'skills', name), { recursive: true }); + await writeFile(path.join(source, 'skills', name, 'SKILL.md'), `---\nname: ${name}\ndescription: >\n A Skill.\n---\n# Instructions`); + } + await buildBundle(source, bundle, revision); + return { root, source, bundle, workspace }; +} + +test('all source Skills are bundled without a name or count allowlist', async t => { + const f = await fixture(t, ['new-tool', 'another-tool']); + assert.deepEqual((await verifyBundle(f.bundle)).skills, ['another-tool', 'new-tool']); + assert.equal((await prepareBundle(f.bundle, f.workspace)).skillCount, 2); +}); + +test('dirty build output cannot produce a successful but invalid bundle', async t => { + const f = await fixture(t); + await mkdir(path.join(f.bundle, 'skills/stale')); + await writeFile(path.join(f.bundle, 'skills/stale/SKILL.md'), 'stale'); + await buildBundle(f.source, f.bundle, revision); + assert.deepEqual((await verifyBundle(f.bundle)).skills, ['sealos-deploy']); +}); + +test('retry restores unrelated Skills from a legacy interrupted replacement', async t => { + const f = await fixture(t); + await prepareBundle(f.bundle, f.workspace); + const target = path.join(f.workspace, '.agents/skills'); + await mkdir(path.join(target, 'custom')); + await writeFile(path.join(target, 'custom/keep'), 'user content'); + await rename(target, path.join(f.workspace, '.agents/skills-backup-99999')); + await mkdir(path.join(f.workspace, '.agents/skills-stage-99999')); + assert.equal((await prepareBundle(f.bundle, f.workspace)).status, 'ready'); + assert.equal(await readFile(path.join(target, 'custom/keep'), 'utf8'), 'user content'); +}); + +test('deployment preparation requires an existing workspace', async t => { + const f = await fixture(t); + await assert.rejects(prepareBundle(f.bundle, path.join(f.root, 'not-cloned'))); +}); + +test('CLI invocation through a symlink must execute, not exit with empty success', async t => { + const f = await fixture(t); + const alias = path.join(f.root, 'bundle-cli.mjs'); + await symlink(new URL('./skill-bundle.mjs', import.meta.url).pathname, alias); + const result = spawnSync(process.execPath, [alias, 'verify', f.bundle], { encoding: 'utf8' }); + assert.equal(result.status, 0); + assert.equal(JSON.parse(result.stdout).revision, revision); +}); + +for (const checkpoint of ['backup', 'published']) { + test(`SIGKILL after ${checkpoint} preserves custom Skills on retry`, async t => { + const f = await fixture(t); + await prepareBundle(f.bundle, f.workspace); + const target = path.join(f.workspace, '.agents/skills'); + await mkdir(path.join(target, 'custom')); + await writeFile(path.join(target, 'custom/keep'), 'user content'); + const child = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import fs from 'node:fs'; + import { syncBuiltinESMExports } from 'node:module'; + const rename = fs.promises.rename; + fs.promises.rename = async (from, to) => { + await rename(from, to); + if (${JSON.stringify(checkpoint)} === 'backup' ? to.endsWith('/backup') : from.endsWith('/stage')) process.kill(process.pid, 'SIGKILL'); + }; + syncBuiltinESMExports(); + const { prepareBundle } = await import(${JSON.stringify(new URL('./skill-bundle.mjs', import.meta.url).href)}); + await prepareBundle(${JSON.stringify(f.bundle)}, ${JSON.stringify(f.workspace)}); + `], { encoding: 'utf8' }); + assert.equal(child.signal, 'SIGKILL', child.stderr); + assert.equal((await prepareBundle(f.bundle, f.workspace)).status, 'ready'); + assert.equal(await readFile(path.join(target, 'custom/keep'), 'utf8'), 'user content'); + }); +} + +test('Chat explicitly creates its workspace without weakening the deployment precondition', async t => { + const f = await fixture(t); + const workspace = path.join(f.root, 'chat'); + const child = spawnSync(process.execPath, [new URL('./skill-bundle.mjs', import.meta.url).pathname, 'prepare-chat', f.bundle, workspace], { encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); + assert.equal(validateReady(child.stdout).status, 'ready'); +}); + +test('success output must match the ready contract', () => { + for (const raw of ['', '{}', 'null', '{"schema":1,"status":"ready"}', '{"status":"failed"}']) { + assert.throws(() => validateReady(raw), /invalid_ready/); + } + const ready = { schema: 1, status: 'ready', revision, digest: 'c'.repeat(64), skillCount: 4 }; + assert.deepEqual(validateReady(JSON.stringify({ ...ready, untrusted: 'discard' })), ready); +}); + +test('preparation does not read or hash workspace file bodies', async t => { + const f = await fixture(t); + await prepareBundle(f.bundle, f.workspace); + const child = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import fs from 'node:fs'; + import { syncBuiltinESMExports } from 'node:module'; + const readFile = fs.promises.readFile; + fs.promises.readFile = async (file, options) => { + if (String(file).startsWith(${JSON.stringify(f.workspace)})) throw new Error('workspace content read'); + return readFile(file, options); + }; + syncBuiltinESMExports(); + const { prepareBundle } = await import(${JSON.stringify(new URL('./skill-bundle.mjs', import.meta.url).href)}); + console.log(JSON.stringify(await prepareBundle(${JSON.stringify(f.bundle)}, ${JSON.stringify(f.workspace)}))); + `], { encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); + assert.equal(validateReady(child.stdout).status, 'ready'); +}); + +test('ambiguous legacy backups fail closed without hiding preserved content', async t => { + const f = await fixture(t); + const agent = path.join(f.workspace, '.agents'); + for (const id of ['1', '2']) { + await mkdir(path.join(agent, 'skills-backup-' + id), { recursive: true }); + await writeFile(path.join(agent, 'skills-backup-' + id, 'keep'), id); + } + await assert.rejects(prepareBundle(f.bundle, f.workspace), /recovery_ambiguous/); + assert.equal(await readFile(path.join(agent, 'skills-backup-1/keep'), 'utf8'), '1'); + await assert.rejects(readFile(path.join(agent, 'skills/SKILL.md'))); +}); + +test('backup cleanup failure after publication does not turn readiness into failure', async t => { + const f = await fixture(t); + const child = spawnSync(process.execPath, ['--input-type=module', '-e', ` + import fs from 'node:fs'; + import { syncBuiltinESMExports } from 'node:module'; + const rm = fs.promises.rm; + fs.promises.rm = async (entry, options) => { + if (entry.split('/').at(-1).startsWith('txn-')) throw new Error('cleanup failure'); + return rm(entry, options); + }; + syncBuiltinESMExports(); + const { prepareBundle } = await import(${JSON.stringify(new URL('./skill-bundle.mjs', import.meta.url).href)}); + console.log(JSON.stringify(await prepareBundle(${JSON.stringify(f.bundle)}, ${JSON.stringify(f.workspace)}))); + `], { encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); + assert.equal(validateReady(child.stdout).status, 'ready'); + assert.equal((await prepareBundle(f.bundle, f.workspace)).status, 'ready'); +}); diff --git a/base-images/frameworks/sandbox/v1/skill-bundle.mjs b/base-images/frameworks/sandbox/v1/skill-bundle.mjs new file mode 100644 index 00000000..13b3a9f2 --- /dev/null +++ b/base-images/frameworks/sandbox/v1/skill-bundle.mjs @@ -0,0 +1,191 @@ +import { createHash } from 'node:crypto'; +import { cp, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const hash = data => createHash('sha256').update(data).digest('hex'); +const namePattern = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const fail = code => { throw new Error(code); }; + +async function files(root, prefix = '', hashes = true) { + if (!(await lstat(path.join(root, prefix))).isDirectory()) fail('bundle_symlink'); + const result = {}; + for (const entry of (await readdir(path.join(root, prefix))).sort()) { + const relative = path.posix.join(prefix, entry); + const stat = await lstat(path.join(root, relative)); + if (stat.isSymbolicLink()) fail('bundle_symlink'); + if (stat.isDirectory()) Object.assign(result, await files(root, relative, hashes)); + else if (stat.isFile()) { + if (hashes) result[relative] = hash(await readFile(path.join(root, relative))); + } + else fail('bundle_special_file'); + } + return result; +} + +export async function buildBundle(source, destination, revision) { + if (!/^[a-f0-9]{40}$/.test(revision)) fail('invalid_revision'); + source = await realpath(source); + destination = path.resolve(destination); + if ([path.parse(destination).root, os.homedir(), process.cwd()].includes(destination) || + source === destination || source.startsWith(destination + path.sep) || + destination.startsWith(source + path.sep)) fail('invalid_destination'); + await safeDirectory(path.dirname(destination)); + if (await exists(destination)) { + await safeDirectory(destination); + const entries = await readdir(destination); + if (entries.some(name => !['manifest.json', 'skills'].includes(name)) || + (entries.length && !entries.includes('manifest.json'))) fail('invalid_destination'); + await files(destination, '', false); + } + const sourceSkills = path.join(source, 'skills'); + const names = []; + for (const entry of (await readdir(sourceSkills)).sort()) { + if (!namePattern.test(entry)) fail('invalid_skill_name'); + const directory = path.join(sourceSkills, entry); + if (!(await lstat(directory)).isDirectory()) fail('invalid_skill_directory'); + const content = await readFile(path.join(directory, 'SKILL.md'), 'utf8'); + const frontmatter = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); + if (!frontmatter || !/^name:[ \t]*\S[^\r\n]*$/m.test(frontmatter[1]) || !/^description:[ \t]*\S[^\r\n]*$/m.test(frontmatter[1])) fail('invalid_skill_metadata'); + await files(directory, '', false); + names.push(entry); + } + if (!names.length) fail('empty_bundle'); + const staging = await mkdtemp(path.join(path.dirname(destination), '.skill-build-')); + try { + await mkdir(path.join(staging, 'skills')); + for (const name of names) await cp(path.join(sourceSkills, name), path.join(staging, 'skills', name), { recursive: true }); + const manifest = { schema: 1, revision, skills: names, files: await files(path.join(staging, 'skills')) }; + await writeFile(path.join(staging, 'manifest.json'), JSON.stringify(manifest)); + await verifyBundle(staging); + await rm(destination, { recursive: true, force: true }); + await rename(staging, destination); + return (await verifyBundle(destination)).digest; + } finally { + await rm(staging, { recursive: true, force: true }); + } +} + +export async function verifyBundle(bundle) { + const raw = await readFile(path.join(bundle, 'manifest.json'), 'utf8'); + const manifest = JSON.parse(raw); + if (manifest?.schema !== 1 || !/^[a-f0-9]{40}$/.test(manifest.revision) || !Array.isArray(manifest.skills) || !manifest.skills.length || !manifest.skills.every(name => typeof name === 'string' && namePattern.test(name))) fail('invalid_manifest'); + const actual = await files(path.join(bundle, 'skills')); + if (JSON.stringify(actual) !== JSON.stringify(manifest.files)) fail('bundle_integrity_failed'); + if (manifest.skills.some(name => !actual[`${name}/SKILL.md`])) fail('missing_skill'); + if (new Set(manifest.skills).size !== manifest.skills.length || Object.keys(actual).some(file => !manifest.skills.includes(file.split('/')[0]))) fail('invalid_manifest'); + if (JSON.stringify((await readdir(path.join(bundle, 'skills'))).sort()) !== JSON.stringify([...manifest.skills].sort())) fail('invalid_manifest'); + return { ...manifest, digest: hash(raw) }; +} + +async function safeDirectory(directory, create = false) { + const parent = path.dirname(directory); + if (parent !== directory) await safeDirectory(parent); + if (create) await mkdir(directory).catch(error => { if (error.code !== 'EEXIST') throw error; }); + if (!(await lstat(directory)).isDirectory()) fail('workspace_symlink'); +} + +const exists = async entry => { + try { await lstat(entry); return true; } + catch (error) { if (error.code === 'ENOENT') return false; throw error; } +}; +const cleanup = directory => rm(directory, { recursive: true, force: true }).catch(() => {}); + +// The caller holds flock. Backup presence is the recovery journal: +// absent target + backup => restore old; present target => publication completed. +async function recoverTransactions(state, target) { + const transactions = (await readdir(state)).filter(name => name.startsWith('txn-')); + if (transactions.length > 1) fail('recovery_ambiguous'); + for (const name of transactions) { + const transaction = path.join(state, name); + await files(transaction, '', false); + const backup = path.join(transaction, 'backup'); + if (!(await exists(target))) { + if (!(await exists(backup))) fail('recovery_ambiguous'); + await rename(backup, target); + } + await cleanup(transaction); + if (await exists(transaction)) fail('recovery_cleanup_failed'); + } +} + +async function recoverLegacy(agentRoot, target) { + const entries = await readdir(agentRoot); + const backups = entries.filter(name => /^skills-backup-\d+$/.test(name)); + const stages = entries.filter(name => /^skills-stage-\d+$/.test(name)); + if (!backups.length && !stages.length) return; + if (backups.length !== 1 || await exists(target)) fail('recovery_ambiguous'); + const backup = path.join(agentRoot, backups[0]); + await files(backup, '', false); + await rename(backup, target); + // A stage may be incomplete. Never promote it over the previous user's tree. + for (const name of stages) { + const stage = path.join(agentRoot, name); + await files(stage, '', false); + await cleanup(stage); + } +} + +export async function prepareBundle(bundle, workspace) { + const manifest = await verifyBundle(bundle); + await safeDirectory(workspace); + const agentRoot = path.join(workspace, '.agents'); + await safeDirectory(agentRoot, true); + const target = path.join(agentRoot, 'skills'); + const state = path.join(workspace, '.sealai-skill-transactions'); + await safeDirectory(state, true); + await recoverTransactions(state, target); + await recoverLegacy(agentRoot, target); + await safeDirectory(target, true); + await files(target, '', false); // Type scan only; do not hash user content. + const transaction = await mkdtemp(path.join(state, 'txn-')); + const stage = path.join(transaction, 'stage'); + const backup = path.join(transaction, 'backup'); + try { + await cp(target, stage, { recursive: true }); + for (const name of manifest.skills) { + await rm(path.join(stage, name), { recursive: true, force: true }); + await cp(path.join(bundle, 'skills', name), path.join(stage, name), { recursive: true }); + } + await rename(target, backup); + await rename(stage, target); + } catch (error) { + // If restoration throws, retain the transaction for the next invocation. + if (await exists(backup) && !(await exists(target))) await rename(backup, target); + await cleanup(transaction); + throw error; + } + await cleanup(transaction); // Publication succeeded; cleanup is best-effort. + return { schema: 1, status: 'ready', revision: manifest.revision, digest: manifest.digest, skillCount: manifest.skills.length }; +} + +export function validateReady(raw) { + let value; + try { value = JSON.parse(raw); } catch { fail('invalid_ready'); } + if (value?.schema !== 1 || value.status !== 'ready' || + !/^[a-f0-9]{40}$/.test(value.revision) || !/^[a-f0-9]{64}$/.test(value.digest) || + !Number.isSafeInteger(value.skillCount) || value.skillCount < 1) fail('invalid_ready'); + return { schema: 1, status: 'ready', revision: value.revision, digest: value.digest, skillCount: value.skillCount }; +} + +if (process.argv[1] && await realpath(process.argv[1]) === fileURLToPath(import.meta.url)) { + const [operation, ...args] = process.argv.slice(2); + try { + let result; + if (operation === 'build') result = await buildBundle(...args); + else if (operation === 'verify') result = await verifyBundle(...args); + else if (operation === 'prepare' || operation === 'prepare-chat') { + if (operation === 'prepare-chat') await safeDirectory(args[1], true); + result = await prepareBundle(...args); + } else if (operation === 'validate-ready') { + let raw = ''; + for await (const chunk of process.stdin) raw += chunk; + result = validateReady(raw); + } else fail('invalid_operation'); + console.log(JSON.stringify(result)); + } catch { + console.error(JSON.stringify({ schema: 1, status: 'failed', reason: 'skill_bundle_unavailable' })); + process.exitCode = 1; + } +} diff --git a/base-images/frameworks/sandbox/v1/skill-bundle.test.mjs b/base-images/frameworks/sandbox/v1/skill-bundle.test.mjs new file mode 100644 index 00000000..6ab1dbef --- /dev/null +++ b/base-images/frameworks/sandbox/v1/skill-bundle.test.mjs @@ -0,0 +1,107 @@ +import assert from 'node:assert/strict'; +import { cp, mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { buildBundle, prepareBundle, verifyBundle } from './skill-bundle.mjs'; + +const revision = 'a'.repeat(40); +async function fixture(t) { + const root = await mkdtemp(path.join(await realpath(os.tmpdir()), 'skill-bundle-test-')); + t.after(() => rm(root, { recursive: true, force: true })); + const source = path.join(root, 'source'); + const bundle = path.join(root, 'bundle'); + const workspace = path.join(root, 'workspace'); + await mkdir(workspace); + const skill = path.join(source, 'skills/sealos-deploy'); + await mkdir(path.join(skill, 'references'), { recursive: true }); + await writeFile(path.join(skill, 'SKILL.md'), '---\nname: sealos-deploy\ndescription: Deploy applications.\n---\n# Deploy'); + await writeFile(path.join(skill, 'references/guide.md'), 'offline guide'); + await buildBundle(source, bundle, revision); + return { root, source, bundle, workspace, skill }; +} + +test('offline preparation is repeatable, copies resources and preserves unrelated project files', async t => { + const f = await fixture(t); + const custom = path.join(f.workspace, '.agents/skills/custom'); + await mkdir(custom, { recursive: true }); + await writeFile(path.join(custom, 'SKILL.md'), 'user skill'); + await writeFile(path.join(f.workspace, 'skills-lock.json'), 'user lock'); + const first = await prepareBundle(f.bundle, f.workspace); + assert.equal(first.status, 'ready'); + assert.equal(first.revision, revision); + assert.equal(first.skillCount, 1); + assert.match(first.digest, /^[a-f0-9]{64}$/); + assert.deepEqual(await prepareBundle(f.bundle, f.workspace), first); + assert.equal(await readFile(path.join(f.workspace, '.agents/skills/sealos-deploy/references/guide.md'), 'utf8'), 'offline guide'); + assert.equal(await readFile(path.join(custom, 'SKILL.md'), 'utf8'), 'user skill'); + assert.equal(await readFile(path.join(f.workspace, 'skills-lock.json'), 'utf8'), 'user lock'); +}); + +test('canonical plugin directory builds without following repository skill aliases', async t => { + const f = await fixture(t); + const repo = path.join(f.root, 'plugin-repo'); + const plugin = path.join(repo, 'plugins/sealos'); + await mkdir(plugin, { recursive: true }); + await cp(f.source, plugin, { recursive: true }); + await mkdir(path.join(repo, 'skills')); + await symlink('../plugins/sealos/skills/sealos-deploy', path.join(repo, 'skills/sealos-deploy')); + const bundle = path.join(f.root, 'plugin-bundle'); + await buildBundle(plugin, bundle, revision); + assert.equal((await prepareBundle(bundle, f.workspace)).skillCount, 1); + assert.equal(await readFile(path.join(f.workspace, '.agents/skills/sealos-deploy/references/guide.md'), 'utf8'), 'offline guide'); + await assert.rejects(buildBundle(repo, path.join(f.root, 'alias-bundle'), revision), /invalid_skill_directory/); +}); + +test('tampered bundle fails before touching workspace', async t => { + const f = await fixture(t); + await prepareBundle(f.bundle, f.workspace); + await writeFile(path.join(f.bundle, 'skills/sealos-deploy/SKILL.md'), 'corrupt'); + await assert.rejects(prepareBundle(f.bundle, f.workspace), /bundle_integrity_failed/); + assert.match(await readFile(path.join(f.workspace, '.agents/skills/sealos-deploy/SKILL.md'), 'utf8'), /# Deploy/); +}); + +test('manifest must enumerate a nonempty, unique set of source skills', async t => { + const f = await fixture(t); + const file = path.join(f.bundle, 'manifest.json'); + const manifest = JSON.parse(await readFile(file, 'utf8')); + await writeFile(file, JSON.stringify({ ...manifest, skills: [] })); + await assert.rejects(verifyBundle(f.bundle), /invalid_manifest/); + await writeFile(file, JSON.stringify({ ...manifest, skills: ['sealos-deploy', 'sealos-deploy'] })); + await assert.rejects(verifyBundle(f.bundle), /invalid_manifest/); +}); + +test('repository symlinks cannot redirect Skill writes', async t => { + const f = await fixture(t); + const outside = path.join(f.root, 'outside'); + await mkdir(outside); + await writeFile(path.join(outside, 'keep'), 'unchanged'); + await symlink(outside, path.join(f.workspace, '.agents')); + await assert.rejects(prepareBundle(f.bundle, f.workspace), /workspace_symlink/); + assert.equal(await readFile(path.join(outside, 'keep'), 'utf8'), 'unchanged'); +}); + +test('nested workspace symlinks and bundle symlinks are rejected', async t => { + const f = await fixture(t); + const target = path.join(f.workspace, '.agents/skills/custom'); + await mkdir(target, { recursive: true }); + await symlink(f.skill, path.join(target, 'link')); + await assert.rejects(prepareBundle(f.bundle, f.workspace), /bundle_symlink/); + await symlink(f.skill, path.join(f.bundle, 'skills/link')); + await assert.rejects(verifyBundle(f.bundle), /bundle_symlink/); +}); + +test('invalid source metadata and non-commit revisions fail at build time', async t => { + const f = await fixture(t); + await assert.rejects(buildBundle(f.source, path.join(f.root, 'other'), 'main'), /invalid_revision/); + await writeFile(path.join(f.skill, 'SKILL.md'), '# no metadata'); + await assert.rejects(buildBundle(f.source, path.join(f.root, 'other'), revision), /invalid_skill_metadata/); +}); + +test('staging failure leaves the previous workspace intact', async t => { + const f = await fixture(t); + await prepareBundle(f.bundle, f.workspace); + await mkdir(path.join(f.workspace, `.agents/skills-backup-${process.pid}`)); + await assert.rejects(prepareBundle(f.bundle, f.workspace), /recovery_ambiguous/); + assert.match(await readFile(path.join(f.workspace, '.agents/skills/sealos-deploy/SKILL.md'), 'utf8'), /# Deploy/); +}); diff --git a/base-images/frameworks/sandbox/v1/verify-pinned-source.mjs b/base-images/frameworks/sandbox/v1/verify-pinned-source.mjs new file mode 100644 index 00000000..287d4e6c --- /dev/null +++ b/base-images/frameworks/sandbox/v1/verify-pinned-source.mjs @@ -0,0 +1,39 @@ +// Network integration test: read the exact image inputs, not a second test pin. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtemp, mkdir, readFile, readdir, realpath, rm } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { buildBundle, prepareBundle, verifyBundle } from './skill-bundle.mjs'; + +const dockerfile = await readFile(new URL('./Dockerfile', import.meta.url), 'utf8'); +const repository = dockerfile.match(/^ARG SEALOS_SKILLS_REPOSITORY=(.+)$/m)?.[1]; +const revision = dockerfile.match(/^ARG SEALOS_SKILLS_REVISION=([a-f0-9]{40})$/m)?.[1]; +assert.equal(repository, 'https://github.com/norberia/sealos-skills-next.git'); +assert.ok(revision); +const root = await mkdtemp(path.join(await realpath(os.tmpdir()), 'pinned-skill-test-')); +try { + const source = path.join(root, 'source'); + const git = args => execFileSync('git', args, { encoding: 'utf8', timeout: 120_000 }); + git(['init', source]); + git(['-C', source, 'remote', 'add', 'origin', repository]); + git(['-C', source, 'fetch', '--depth', '1', 'origin', revision]); + git(['-C', source, 'checkout', '--detach', 'FETCH_HEAD']); + assert.equal(git(['-C', source, 'rev-parse', 'HEAD']).trim(), revision); + const plugin = path.join(source, 'plugins/sealos'); + const bundle = path.join(root, 'bundle'); + const workspace = path.join(root, 'workspace'); + await mkdir(workspace); + await buildBundle(plugin, bundle, revision); + const manifest = await verifyBundle(bundle); + assert.deepEqual(manifest.skills, (await readdir(path.join(plugin, 'skills'))).sort()); + const first = await prepareBundle(bundle, workspace); + assert.equal(first.skillCount, manifest.skills.length); + assert.deepEqual(await prepareBundle(bundle, workspace), first); + for (const file of Object.keys(manifest.files)) { + assert.deepEqual(await readFile(path.join(workspace, '.agents/skills', file)), await readFile(path.join(plugin, 'skills', file))); + } + console.log(JSON.stringify({ ...first, fileCount: Object.keys(manifest.files).length })); +} finally { + await rm(root, { recursive: true, force: true }); +} diff --git a/tests/runtime-smoke/frameworks/sandbox/v1/smoke.sh b/tests/runtime-smoke/frameworks/sandbox/v1/smoke.sh index 04d2732e..95430514 100755 --- a/tests/runtime-smoke/frameworks/sandbox/v1/smoke.sh +++ b/tests/runtime-smoke/frameworks/sandbox/v1/smoke.sh @@ -90,4 +90,30 @@ if [ ! -d "$workspace_dir/.versitygw-s3/kaniko-contexts/contexts" ]; then exit 1 fi +# Execute only in a disposable smoke-test container, as its actual Devbox user. +test "$(id -un)" = devbox +command -v node +command -v flock +test -x /usr/local/bin/sealai-prepare-skills +node /opt/sealai/skill-bundle.mjs verify /opt/sealai/skill-bundle >/dev/null +mkdir -p /home/devbox/project/.agents/skills/smoke-custom +printf 'preserved\n' > /home/devbox/project/.agents/skills/smoke-custom/SKILL.md +first=$(/usr/local/bin/sealai-prepare-skills) +second=$(/usr/local/bin/sealai-prepare-skills) +test "$first" = "$second" +SKILL_READY_JSON="$second" /usr/bin/node --input-type=module <<'NODE' +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { validateReady, verifyBundle } from '/opt/sealai/skill-bundle.mjs'; +const manifest = await verifyBundle('/opt/sealai/skill-bundle'); +assert.equal(validateReady(process.env.SKILL_READY_JSON).skillCount, manifest.skills.length); +for (const [file, digest] of Object.entries(manifest.files)) { + const data = await readFile('/home/devbox/project/.agents/skills/' + file); + assert.equal(createHash('sha256').update(data).digest('hex'), digest); +} +NODE +test "$(< /home/devbox/project/.agents/skills/smoke-custom/SKILL.md)" = preserved +printf '%s\n' "$second" + echo "ok"