From d0927d427b6445b799097691adfb885d095d5cdf Mon Sep 17 00:00:00 2001 From: NeilJo-GY <43027886+NeilJo-GY@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:58:07 +0800 Subject: [PATCH 1/4] =?UTF-8?q?Add=20persona-seed=20skill=20for=20pluggabl?= =?UTF-8?q?e=20corpus=20=E2=86=92=20persona=20packs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a provider-agnostic seed pipeline (MatrAIx fixture first): search, SeedProfile mapping, generate-gate e2e, and pack-local provenance — so population corpora can feed OpenPersona without one-off per-source create paths. Co-authored-by: Cursor --- skills/open-persona/SKILL.md | 12 +- skills/persona-seed/README.md | 30 ++ skills/persona-seed/SKILL.md | 139 ++++++++ skills/persona-seed/providers/README.md | 14 + .../fixtures/sample-corpus.json | 107 ++++++ .../providers/matraix-persona-1m/mapping.md | 44 +++ .../providers/matraix-persona-1m/provider.js | 304 ++++++++++++++++++ .../references/provider-contract.md | 41 +++ .../persona-seed/schemas/intent.schema.json | 57 ++++ .../schemas/seed-profile.schema.json | 106 ++++++ .../scripts/map-seed-to-persona.js | 224 +++++++++++++ skills/persona-seed/scripts/prepare-corpus.js | 164 ++++++++++ skills/persona-seed/scripts/run-pipeline.js | 133 ++++++++ skills/persona-seed/scripts/search.js | 79 +++++ .../persona-seed/scripts/write-provenance.js | 59 ++++ .../persona-seed/tests/persona-seed.test.js | 180 +++++++++++ tests/persona-seed.test.js | 4 + 17 files changed, 1693 insertions(+), 4 deletions(-) create mode 100644 skills/persona-seed/README.md create mode 100644 skills/persona-seed/SKILL.md create mode 100644 skills/persona-seed/providers/README.md create mode 100644 skills/persona-seed/providers/matraix-persona-1m/fixtures/sample-corpus.json create mode 100644 skills/persona-seed/providers/matraix-persona-1m/mapping.md create mode 100644 skills/persona-seed/providers/matraix-persona-1m/provider.js create mode 100644 skills/persona-seed/references/provider-contract.md create mode 100644 skills/persona-seed/schemas/intent.schema.json create mode 100644 skills/persona-seed/schemas/seed-profile.schema.json create mode 100644 skills/persona-seed/scripts/map-seed-to-persona.js create mode 100644 skills/persona-seed/scripts/prepare-corpus.js create mode 100644 skills/persona-seed/scripts/run-pipeline.js create mode 100644 skills/persona-seed/scripts/search.js create mode 100644 skills/persona-seed/scripts/write-provenance.js create mode 100644 skills/persona-seed/tests/persona-seed.test.js create mode 100644 tests/persona-seed.test.js diff --git a/skills/open-persona/SKILL.md b/skills/open-persona/SKILL.md index 1ec2cec..9967549 100644 --- a/skills/open-persona/SKILL.md +++ b/skills/open-persona/SKILL.md @@ -27,8 +27,9 @@ You are the meta-skill for creating, installing, updating, and publishing agent | --- | --- | | Framework entry point: create, install, manage, publish, run personas | **`persona-evaluator`** — quality audit (4 Layers × 5 Concepts × Constitution gate) | | Runner integration protocol (`openpersona state` commands) | **`anyone-skill`** — distill a real person or character into a persona pack | -| Skill registry (`openpersona skill` commands) | **`brand-persona-skill`** — turn a commercial entity into a brand agent | -| Evolution, economy, vitality, social, rhythm configuration | **`persona-model-trainer`** — fine-tune a local model on persona data | +| Skill registry (`openpersona skill` commands) | **`persona-seed`** — sample an archetype from a population corpus (e.g. MatrAIx Persona 1M fixture / decoded JSON) | +| Evolution, economy, vitality, social, rhythm configuration | **`brand-persona-skill`** — turn a commercial entity into a brand agent | +| | **`persona-model-trainer`** — fine-tune a local model on persona data | ## What You Can Do @@ -58,13 +59,15 @@ npx openpersona create When a user asks you to create a persona (e.g. "make me a coding mentor", "build a companion persona"), follow this playbook: -### Step 1 — Decide: preset or custom? +### Step 1 — Decide: preset, corpus seed, distillation, or custom? | User request | Action | | ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------- | | Matches an existing preset (`ai-girlfriend`, `life-assistant`, `stoic-mentor`, `samantha`, `health-butler`…) | Use `--preset ` directly — skip to Step 4 | -| Specific role / domain / personality | Gather 3 required inputs (Step 2), then write persona.json (Step 3) | +| Wants a pack sampled from a population corpus / MatrAIx / “pick a seed profile” | Delegate to **`persona-seed`** (`skills/persona-seed/SKILL.md`) — then return here for install/manage | +| Distill a specific real person or named character from evidence | Delegate to **`anyone-skill`** | +| Specific role / domain / personality (author from scratch) | Gather 3 required inputs (Step 2), then write persona.json (Step 3) | ### Step 2 — Gather minimum required inputs (3 questions max) @@ -548,6 +551,7 @@ If an automated scanner flags "suspicious," it is usually because **persona mana | --------------------------------------------------------------------------- | ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | | `[persona-evaluator](https://github.com/acnlabs/persona-evaluator)` | `openpersona skill install acnlabs/persona-evaluator` | Score any persona pack across 4 Layers + 5 Systemic Concepts — 9-dimension quality audit, constitution compliance gate, and actionable improvement recommendations (`npx openpersona evaluate `) | | `[anyone-skill](https://github.com/acnlabs/anyone-skill)` | `openpersona skill install acnlabs/anyone-skill` | Distill any person or character (self, personal, public, historical, fictional, archetype) into a persona skill pack | +| `persona-seed` (in-repo: `skills/persona-seed`) | local / `npx skills add` when published | Sample archetype seeds from pluggable corpora (first provider: MatrAIx Persona 1M) → SeedProfile → `persona.json` → create | | `[brand-persona-skill](https://github.com/acnlabs/brand-persona-skill)` | `openpersona skill install acnlabs/brand-persona-skill` | Turn any commercial entity (shop, clinic, studio, chain) into a brand agent — soul distillation or declaration from scratch, service skills, A2A discoverability, and a service contract | | `[persona-model-trainer](https://github.com/acnlabs/persona-model-trainer)` | `openpersona skill install acnlabs/persona-model-trainer` | Fine-tune Gemma-4 (E2B/E4B) locally on distilled data — self-contained model for phones and laptops via Ollama/llama.cpp. Use `--preset gemma4` for one-command optimised training (lora-rank=16, alpha=rank, lora-layers=16, warmup=0.1). | | `[persona-knowledge](https://github.com/acnlabs/persona-knowledge)` | `openpersona skill install acnlabs/persona-knowledge` | Persistent, searchable persona knowledge base — MemPalace storage + Knowledge Graph + Karpathy LLM Wiki + training/ export | diff --git a/skills/persona-seed/README.md b/skills/persona-seed/README.md new file mode 100644 index 0000000..81208b3 --- /dev/null +++ b/skills/persona-seed/README.md @@ -0,0 +1,30 @@ +# persona-seed + +Population / corpus **seed providers** → OpenPersona persona packs. + +One orchestration skill; pluggable adapters. First provider: **MatrAIx Persona 1M** (fixture by default; optional larger JSON corpus via `MATRAIX_CORPUS_PATH`). + +## When to use + +| Goal | Skill | +|------|--------| +| Distill a specific person / character from evidence | `anyone-skill` | +| Sample an archetype from a persona corpus | **`persona-seed`** | + +## Quick start (agent) + +1. Read `SKILL.md` and follow Phases 0–5. +2. One-shot pipeline (fixture): + `node scripts/run-pipeline.js --intent '{"domain":["software"],"traits":["precise"]}' --name "Nova" --slug nova-seed --out /tmp/nova-seed` +3. Or step-by-step: `search.js` → `--to-seed` → `map-seed-to-persona.js` → `openpersona create` → `write-provenance.js` +4. Larger corpus: decode HF → `prepare-corpus.js --validate` → `export MATRAIX_CORPUS_PATH=…` + +## Layout + +``` +SKILL.md +schemas/ intent + SeedProfile +references/ provider contract +providers/ matraix-persona-1m + … +scripts/ search, map-seed-to-persona, write-provenance +``` diff --git a/skills/persona-seed/SKILL.md b/skills/persona-seed/SKILL.md new file mode 100644 index 0000000..a0160ac --- /dev/null +++ b/skills/persona-seed/SKILL.md @@ -0,0 +1,139 @@ +--- +name: persona-seed +description: "Create an OpenPersona pack from population / corpus seed providers (first: MatrAIx Persona 1M). Searches a pluggable seed index from user intent, maps a SeedProfile to persona.json, then generates via open-persona. Use when the user wants a persona from a large persona corpus, MatrAIx, or archetype sampling — not when distilling a specific real person (use anyone-skill)." +license: MIT +compatibility: "OpenPersona / Cursor / Claude Code / OpenClaw. Node.js >= 18. Optional: decoded MatrAIx corpus via MATRAIX_CORPUS_PATH." +allowed-tools: Read Write Edit Bash +metadata: + version: "0.1.0" + author: acnlabs +--- + +# persona-seed — Population seeds → OpenPersona packs + +> One orchestration skill. Many corpus providers. MatrAIx Persona 1M is the first adapter — not a special-case product path. + +**Dependency chain**: `persona-seed` → `skills/open-persona` → `openpersona create` +**Sibling**: `anyone-skill` (evidence distillation of a specific subject). Prefer anyone-skill for real people / named characters; prefer persona-seed for population sampling and archetypes from a corpus. + +## Trigger phrases + +- `/persona-seed` +- "create a persona from MatrAIx" +- "sample a persona from Persona 1M" +- "pick a seed profile and generate a pack" +- "从人口语料里生成人格包" + +## Architecture + +``` +user intent + → provider.search (attribute filter) + → agent re-rank + user pick + → provider.fetch + toSeed → SeedProfile + → fill gaps (name, role, boundaries…) + → map-seed-to-persona.js → persona.json + → openpersona create --config … + → write-provenance.js → soul/seed-provenance.json +``` + +Contract: `references/provider-contract.md` +Schemas: `schemas/intent.schema.json`, `schemas/seed-profile.schema.json` +Providers: `providers/` (add new sources there only) + +## Tools (scripts) + +Set `SEED_DIR` to this skill directory (path containing `SKILL.md`). + +| Task | Command | +|------|---------| +| List provider caps | `node ${SEED_DIR}/scripts/search.js --capabilities` | +| Search seeds | `node ${SEED_DIR}/scripts/search.js --intent ''` | +| Fetch raw | `node ${SEED_DIR}/scripts/search.js --fetch --id ` | +| To SeedProfile | `node ${SEED_DIR}/scripts/search.js --to-seed --id ` | +| Map to persona.json | `node ${SEED_DIR}/scripts/map-seed-to-persona.js --seed --overrides '' --out persona.json` | +| Write provenance | `node ${SEED_DIR}/scripts/write-provenance.js --pack --seed ` | +| Full pipeline | `node ${SEED_DIR}/scripts/run-pipeline.js --intent '' --name "…" --slug … --role assistant --out ` | +| Validate corpus | `node ${SEED_DIR}/scripts/prepare-corpus.js --validate ` | + +Default corpus is the offline **fixture** (5 rows) under `providers/matraix-persona-1m/fixtures/`. `capabilities()` / provenance report `corpusMode: "fixture"` and a fixture dataset id — not the Hugging Face 1M release. + +To use a larger decoded corpus: decode HF Parquet offline → JSON/JSONL in the fixture record shape → `prepare-corpus.js --validate` → `export MATRAIX_CORPUS_PATH=…`. + +Empty filter results return `[]` (no silent unfiltered fallback). Re-check intent if search is empty. + +## Phase 0 — Route + +If the user wants a **specific real person / named character with evidence** → hand off to `anyone-skill`. +If they want a **sampled archetype / corpus seed** → continue here. + +Confirm provider (default `matraix-persona-1m`). + +## Phase 1 — Intent + +Extract an Intent object (see `schemas/intent.schema.json`): + +- `roleHint`, `domain`, `traits`, `locale`, `region`, `exclude`, `query`, `limit` (default 5) + +Do not invent MatrAIx attribute names for the user; keep intent in this normalized shape. + +## Phase 2 — Search + re-rank + +1. Run `search.js --intent …` (attribute filter scores). +2. If the array is empty, broaden or rewrite intent — do **not** invent hits. +3. **You** re-rank the shortlist against the user's free-text goal. +4. Present 3–5 candidates (summary + highlights + `corpusMode`). Ask the user to pick one (or auto-pick top after confirmation). + +## Phase 3 — SeedProfile + +```bash +node ${SEED_DIR}/scripts/search.js --to-seed --id > /tmp/seed-profile.json +``` + +Show `identity.summary`, key traits, `gaps`, and `constraints.sensitiveFlags`. +Fill gaps with the user: + +- required: `personaName`, `slug`, `role` +- recommended: `boundaries` (especially if `healthcare_domain` is flagged) + +**Never** set `soul.identity.sourceIdentity` for MatrAIx / statistical seeds. + +## Phase 4 — persona.json + create + +```bash +node ${SEED_DIR}/scripts/map-seed-to-persona.js \ + --seed /tmp/seed-profile.json \ + --overrides '{"personaName":"…","slug":"…","role":"mentor"}' \ + --out /tmp/persona.json +``` + +Then follow `skills/open-persona` to generate/install: + +```bash +npx openpersona create --config /tmp/persona.json --output --install +``` + +(or the equivalent non-interactive create flags available in this repo) + +## Phase 5 — Provenance + +After the pack directory exists: + +```bash +node ${SEED_DIR}/scripts/write-provenance.js --pack --seed /tmp/seed-profile.json +``` + +Writes `soul/seed-provenance.json` only — does **not** add unknown root keys to `persona.json`. + +## Ethics (short) + +- Corpus seeds are **archetypes**, not digital twins of living people. +- Human-grounded MatrAIx rows are de-identified statistical profiles — still do not claim “I am that person.” +- Respect dataset license notes from `capabilities()`. +- Healthcare-flagged seeds must keep clinical non-authority boundaries. + +## Adding another corpus later + +1. Add `providers//` implementing the contract. +2. Register in `scripts/search.js` → `PROVIDERS`. +3. No new orchestration skill — reuse this one. diff --git a/skills/persona-seed/providers/README.md b/skills/persona-seed/providers/README.md new file mode 100644 index 0000000..eb2b1fd --- /dev/null +++ b/skills/persona-seed/providers/README.md @@ -0,0 +1,14 @@ +# Seed providers + +Each directory is one **Persona Seed** adapter. Orchestration lives in `../SKILL.md`; providers only implement the contract in `../references/provider-contract.md`. + +| Provider id | Status | Notes | +|-------------|--------|-------| +| `matraix-persona-1m` | v1 | Attribute filter + fixture corpus; set `MATRAIX_CORPUS_PATH` for a larger decoded JSON array | + +## Add a provider + +1. Copy `matraix-persona-1m/` as a template. +2. Implement `capabilities`, `search`, `fetch`, `toSeed` in `provider.js`. +3. Register in `../scripts/search.js` → `PROVIDERS`. +4. Keep a tiny offline fixture for tests. diff --git a/skills/persona-seed/providers/matraix-persona-1m/fixtures/sample-corpus.json b/skills/persona-seed/providers/matraix-persona-1m/fixtures/sample-corpus.json new file mode 100644 index 0000000..deb6272 --- /dev/null +++ b/skills/persona-seed/providers/matraix-persona-1m/fixtures/sample-corpus.json @@ -0,0 +1,107 @@ +[ + { + "id": "fixture-001", + "groundingType": "synthetic", + "attributes": { + "region": "East Asia", + "age_bracket": "25-34", + "primary_language": "Chinese", + "highest_education": "Master's", + "occupation_domain": "Software", + "occupation": "Backend engineer", + "risk_tolerance": "Risk-averse", + "communication_formality": "Neutral", + "personality_traits": ["curious", "precise", "reserved"], + "values": ["craftsmanship", "honesty"], + "motivations": ["mastery", "stability"], + "interests": ["open source", "hiking", "science fiction"], + "speaking_style_hints": ["concise", "asks clarifying questions"], + "tools": ["python", "sql", "git"] + }, + "description": "A careful East Asian software engineer in their late twenties who prefers precise answers, low-risk decisions, and open-source craft." + }, + { + "id": "fixture-002", + "groundingType": "synthetic", + "attributes": { + "region": "North America", + "age_bracket": "35-44", + "primary_language": "English", + "highest_education": "Bachelor's", + "occupation_domain": "Education", + "occupation": "High school teacher", + "risk_tolerance": "Moderate", + "communication_formality": "Warm-formal", + "personality_traits": ["patient", "encouraging", "direct"], + "values": ["growth", "fairness"], + "motivations": ["help others learn"], + "interests": ["history", "coaching", "podcasts"], + "speaking_style_hints": ["Socratic questions", "plain language"], + "tools": ["slides", "learning platforms"] + }, + "description": "A North American teacher who coaches with patience, asks Socratic questions, and keeps language plain." + }, + { + "id": "fixture-003", + "groundingType": "human_grounded", + "attributes": { + "region": "Europe", + "age_bracket": "45-54", + "primary_language": "English", + "highest_education": "PhD", + "occupation_domain": "Healthcare", + "occupation": "Clinical researcher", + "risk_tolerance": "Risk-averse", + "communication_formality": "Formal", + "personality_traits": ["rigorous", "cautious", "empathetic"], + "values": ["evidence", "do-no-harm"], + "motivations": ["scientific integrity"], + "interests": ["biostatistics", "classical music"], + "speaking_style_hints": ["hedges claims", "cites uncertainty"], + "tools": ["R", "literature review"] + }, + "description": "A European clinical researcher who hedges claims, cites uncertainty, and refuses medical authority cosplay." + }, + { + "id": "fixture-004", + "groundingType": "synthetic", + "attributes": { + "region": "Southeast Asia", + "age_bracket": "18-24", + "primary_language": "English", + "highest_education": "Some college", + "occupation_domain": "Commerce", + "occupation": "Indie seller", + "risk_tolerance": "Risk-seeking", + "communication_formality": "Casual", + "personality_traits": ["energetic", "persuasive", "optimistic"], + "values": ["hustle", "community"], + "motivations": ["grow a brand"], + "interests": ["social commerce", "design", "street food"], + "speaking_style_hints": ["upbeat", "emoji-light", "short paragraphs"], + "tools": ["shopify", "canva"] + }, + "description": "A young Southeast Asian indie seller with casual upbeat tone and high risk appetite for commerce experiments." + }, + { + "id": "fixture-005", + "groundingType": "synthetic", + "attributes": { + "region": "North America", + "age_bracket": "25-34", + "primary_language": "English", + "highest_education": "Bachelor's", + "occupation_domain": "Software", + "occupation": "Product manager", + "risk_tolerance": "Moderate", + "communication_formality": "Neutral", + "personality_traits": ["structured", "empathetic", "pragmatic"], + "values": ["user outcomes", "clarity"], + "motivations": ["ship useful products"], + "interests": ["UX research", "systems thinking"], + "speaking_style_hints": ["bullet-friendly", "tradeoff-aware"], + "tools": ["figma", "sql", "notion"] + }, + "description": "A pragmatic product manager who frames tradeoffs clearly and stays oriented to user outcomes." + } +] diff --git a/skills/persona-seed/providers/matraix-persona-1m/mapping.md b/skills/persona-seed/providers/matraix-persona-1m/mapping.md new file mode 100644 index 0000000..3d87f3c --- /dev/null +++ b/skills/persona-seed/providers/matraix-persona-1m/mapping.md @@ -0,0 +1,44 @@ +# matraix-persona-1m → SeedProfile + +MatrAIx Persona 1M stores ~1,290 categorical attributes (packed in the HF release). This adapter maps only a **high-signal subset** into `SeedProfile`. Full 1,290 → OpenPersona dump is intentionally unsupported. + +## Attribute → SeedProfile + +| SeedProfile path | MatrAIx / fixture fields | Notes | +|------------------|--------------------------|-------| +| `identity.summary` | record `description` or composed blurbs | Prefer source NL description | +| `identity.region` | `region` | Soft filter + display | +| `identity.ageBracket` | `age_bracket` | Optional | +| `identity.occupation` | `occupation` | Optional | +| `identity.education` | `highest_education` | Optional | +| `identity.locale` | `primary_language` | Normalized to short tags when possible | +| `character.traits` | `personality_traits` | Cap ~12 | +| `character.values` | `values` | | +| `character.motivations` | `motivations` | | +| `character.speakingHints` | `speaking_style_hints` | Hints only; mapper writes final `speakingStyle` | +| `character.interests` | `interests` | | +| `character.riskTolerance` | `risk_tolerance` | Pass-through string | +| `character.formalityBaseline` | `communication_formality` | Mapped to rough -5..5 | +| `capabilities.domains` | `occupation_domain` | Array of one+ | +| `capabilities.skills` / `tools` | `tools` + occupation keywords | Suggestions only | +| `constraints.sensitiveFlags` | healthcare / political / etc. domains | When domain is Healthcare → flag | +| `constraints.suggestedImmutableTraits` | defaults | Always include `honest` at minimum | +| `evidence.attributeHighlights` | selected decoded field=value | Audit trail | +| `gaps` | — | Always include `personaName`, `slug`, `role`, `boundaries` | + +## Defaults / non-goals + +- Do **not** set OpenPersona `sourceIdentity` (archetype / synthetic seed). +- Do **not** invent `personaName` in the adapter; leave for agent + user. +- Full HF parquet decode is v2; v1 uses fixtures or a decoded JSON corpus path via `MATRAIX_CORPUS_PATH`. + +## Dataset / corpusMode + +| Mode | When | `provenance.dataset` | +|------|------|----------------------| +| `fixture` | default (no `MATRAIX_CORPUS_PATH`) | `persona-seed/providers/matraix-persona-1m/fixtures/sample-corpus` | +| `external` | `MATRAIX_CORPUS_PATH` set | `MatrAIx2026/MatrAIx_Persona_1M` (caller-supplied decoded rows) | + +- Public coreset on HF: `MatrAIx2026/MatrAIx_Persona_1M` (~1M rows) — not loaded until decoded into the fixture-shaped JSON array +- Full 8.3B: controlled access — out of scope for this provider id +- Search never falls back to an unfiltered corpus when filters match nothing (`[]`) diff --git a/skills/persona-seed/providers/matraix-persona-1m/provider.js b/skills/persona-seed/providers/matraix-persona-1m/provider.js new file mode 100644 index 0000000..4db5d4e --- /dev/null +++ b/skills/persona-seed/providers/matraix-persona-1m/provider.js @@ -0,0 +1,304 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const PROVIDER_ID = 'matraix-persona-1m'; +const DEFAULT_CORPUS = path.join(__dirname, 'fixtures', 'sample-corpus.json'); +const FIXTURE_DATASET = 'persona-seed/providers/matraix-persona-1m/fixtures/sample-corpus'; +const PUBLIC_DATASET = 'MatrAIx2026/MatrAIx_Persona_1M'; + +const FORMALITY_MAP = { + Casual: -3, + 'Warm-formal': 1, + Neutral: 0, + Formal: 3, +}; + +/** @type {{ path: string, mtimeMs: number, data: object[] } | null} */ +let corpusCache = null; + +function corpusPath() { + return process.env.MATRAIX_CORPUS_PATH + ? path.resolve(process.env.MATRAIX_CORPUS_PATH) + : DEFAULT_CORPUS; +} + +function isFixtureCorpus() { + return corpusPath() === path.resolve(DEFAULT_CORPUS); +} + +function resetCorpusCache() { + corpusCache = null; +} + +function loadCorpus() { + const resolved = corpusPath(); + const stat = fs.statSync(resolved); + if ( + corpusCache && + corpusCache.path === resolved && + corpusCache.mtimeMs === stat.mtimeMs + ) { + return corpusCache.data; + } + + const raw = fs.readFileSync(resolved, 'utf8'); + const data = JSON.parse(raw); + if (!Array.isArray(data)) { + throw new Error(`matraix-persona-1m: corpus must be a JSON array: ${resolved}`); + } + corpusCache = { path: resolved, mtimeMs: stat.mtimeMs, data }; + return data; +} + +function asArray(v) { + if (v == null) return []; + return Array.isArray(v) ? v : [v]; +} + +function norm(s) { + return String(s || '') + .toLowerCase() + .trim(); +} + +function haystack(record) { + const a = record.attributes || {}; + return [ + record.description, + a.region, + a.age_bracket, + a.primary_language, + a.occupation, + a.occupation_domain, + a.highest_education, + a.risk_tolerance, + ...asArray(a.personality_traits), + ...asArray(a.values), + ...asArray(a.motivations), + ...asArray(a.interests), + ...asArray(a.speaking_style_hints), + ...asArray(a.tools), + ] + .map(norm) + .join(' '); +} + +function hasPositiveFilters(intent) { + return ['traits', 'domain', 'region', 'locale', 'ageBracket'].some( + (k) => asArray(intent[k]).length > 0 + ); +} + +function isExcluded(text, intent) { + return asArray(intent.exclude).some((ex) => ex && text.includes(norm(ex))); +} + +function scoreRecord(record, intent) { + const text = haystack(record); + const a = record.attributes || {}; + let score = 0; + const hits = []; + + for (const t of asArray(intent.traits)) { + if (t && text.includes(norm(t))) { + score += 3; + hits.push(`trait:${t}`); + } + } + for (const d of asArray(intent.domain)) { + if (d && text.includes(norm(d))) { + score += 3; + hits.push(`domain:${d}`); + } + } + for (const r of asArray(intent.region)) { + if (r && norm(a.region).includes(norm(r))) { + score += 2; + hits.push(`region:${r}`); + } + } + for (const loc of asArray(intent.locale)) { + if (loc && text.includes(norm(loc))) { + score += 2; + hits.push(`locale:${loc}`); + } + } + for (const age of asArray(intent.ageBracket)) { + if (age && norm(a.age_bracket) === norm(age)) { + score += 1; + hits.push(`age:${age}`); + } + } + + // Free-text tokens (lightweight attribute boost before agent rerank) + if (intent.query) { + for (const tok of norm(intent.query) + .split(/[^a-z0-9\u4e00-\u9fff]+/) + .filter((x) => x.length > 2)) { + if (text.includes(tok)) { + score += 1; + hits.push(`query:${tok}`); + } + } + } + + return { score, hits, text }; +} + +function toCandidate(record, score, hits) { + const a = record.attributes || {}; + return { + id: record.id, + provider: PROVIDER_ID, + summary: + record.description || `${a.occupation || 'persona'} · ${a.region || 'unknown region'}`, + highlights: { + region: a.region || null, + traits: asArray(a.personality_traits).slice(0, 5), + domain: asArray(a.occupation_domain), + groundingType: record.groundingType || 'unknown', + corpusMode: isFixtureCorpus() ? 'fixture' : 'external', + hits, + }, + score, + }; +} + +function capabilities() { + const fixture = isFixtureCorpus(); + return { + id: PROVIDER_ID, + name: 'MatrAIx Persona 1M', + dataset: fixture ? FIXTURE_DATASET : PUBLIC_DATASET, + corpusMode: fixture ? 'fixture' : 'external', + licenseNotes: fixture + ? 'Offline fixture (5 sample rows) for development. Not the Hugging Face 1M release. Set MATRAIX_CORPUS_PATH to a decoded JSON array for external corpus search.' + : 'External decoded corpus path. Prefer rows derived from MatrAIx2026/MatrAIx_Persona_1M; underlying source licenses still apply. Use as archetype seed, not digital twin.', + offline: fixture, + corpusPath: corpusPath(), + }; +} + +/** + * Attribute filter search. + * - `exclude` hard-removes matches (never returned). + * - Positive filters (traits/domain/region/locale/ageBracket) require score > 0. + * - Empty filters / query-only: browse mode (may include score 0). + * - Never falls back to an unfiltered corpus when filters match nothing. + */ +function search(intent = {}) { + const limit = Math.min(Math.max(intent.limit || 5, 1), 20); + const corpus = loadCorpus(); + const requirePositive = hasPositiveFilters(intent); + const ranked = []; + + for (const record of corpus) { + const { score, hits, text } = scoreRecord(record, intent); + if (isExcluded(text, intent)) continue; + if (requirePositive && score <= 0) continue; + ranked.push(toCandidate(record, score, hits)); + } + + ranked.sort((x, y) => y.score - x.score); + return ranked.slice(0, limit); +} + +function fetch(id) { + const record = loadCorpus().find((r) => r.id === id); + if (!record) throw new Error(`matraix-persona-1m: record not found: ${id}`); + return record; +} + +function localeFromLanguage(lang) { + if (!lang) return []; + const n = norm(lang); + if (n.includes('chinese') || n === 'zh' || n.includes('中文')) return ['zh']; + if (n.includes('english') || n === 'en') return ['en']; + if (n.includes('spanish') || n === 'es') return ['es']; + return [lang]; +} + +function toSeed(raw) { + const a = raw.attributes || {}; + const domain = asArray(a.occupation_domain); + const sensitiveFlags = []; + if (domain.some((d) => /health|medical|clinical/i.test(d))) { + sensitiveFlags.push('healthcare_domain'); + } + + const highlights = []; + for (const [field, value] of Object.entries(a)) { + if (value == null) continue; + if (Array.isArray(value)) { + if (value.length) highlights.push({ field, value: value.join(', ') }); + } else { + highlights.push({ field, value: String(value) }); + } + } + + const fixture = isFixtureCorpus(); + + return { + schemaVersion: '0.1.0', + provenance: { + provider: PROVIDER_ID, + recordId: raw.id, + dataset: fixture ? FIXTURE_DATASET : PUBLIC_DATASET, + corpusMode: fixture ? 'fixture' : 'external', + licenseNotes: fixture + ? 'Offline fixture row for development — not a MatrAIx Persona 1M Hugging Face record id.' + : 'External corpus row. Source licenses apply. Archetype only — not a real-person twin.', + groundingType: raw.groundingType || 'unknown', + retrievedAt: new Date().toISOString(), + }, + identity: { + suggestedName: null, + suggestedSlug: null, + summary: raw.description || 'Seed profile from MatrAIx Persona 1M.', + locale: localeFromLanguage(a.primary_language), + region: a.region || null, + ageBracket: a.age_bracket || null, + occupation: a.occupation || null, + education: a.highest_education || null, + }, + character: { + traits: asArray(a.personality_traits), + values: asArray(a.values), + motivations: asArray(a.motivations), + speakingHints: asArray(a.speaking_style_hints), + interests: asArray(a.interests), + riskTolerance: a.risk_tolerance || null, + formalityBaseline: + a.communication_formality != null + ? FORMALITY_MAP[a.communication_formality] ?? 0 + : null, + }, + capabilities: { + domains: domain, + skills: asArray(a.tools), + tools: asArray(a.tools), + }, + constraints: { + hardExclusions: [], + sensitiveFlags, + suggestedImmutableTraits: ['honest', 'helpful'], + }, + evidence: { + attributeHighlights: highlights.slice(0, 24), + rawDescriptions: raw.description ? [raw.description] : [], + }, + gaps: ['personaName', 'slug', 'role', 'boundaries'], + }; +} + +module.exports = { + PROVIDER_ID, + capabilities, + search, + fetch, + toSeed, + resetCorpusCache, + isFixtureCorpus, +}; diff --git a/skills/persona-seed/references/provider-contract.md b/skills/persona-seed/references/provider-contract.md new file mode 100644 index 0000000..29e0d96 --- /dev/null +++ b/skills/persona-seed/references/provider-contract.md @@ -0,0 +1,41 @@ +# Persona Seed — Provider Contract + +All corpus / population sources implement the same interface. MatrAIx Persona 1M is the first provider; new sources add an adapter under `providers//` only. + +## Interface + +| Method | Input | Output | +|--------|-------|--------| +| `capabilities()` | — | `{ id, name, licenseNotes, offline?, dataset? }` | +| `search(intent)` | Intent object (`schemas/intent.schema.json`) | `Candidate[]` (3–10 summaries) | +| `fetch(id)` | provider-local record id | raw record (provider-shaped) | +| `toSeed(raw)` | raw record | `SeedProfile` (`schemas/seed-profile.schema.json`) | + +### Candidate shape + +```json +{ + "id": "provider-local-id", + "provider": "matraix-persona-1m", + "summary": "One-line human-readable blurb", + "highlights": { "region": "…", "traits": ["…"], "domain": ["…"] }, + "score": 0.0 +} +``` + +`score` is the **attribute-filter** score. The orchestrating agent may re-rank candidates against the user's free-text request before asking the user to pick. + +## Rules + +1. Providers never write `persona.json` or call the generator. +2. Providers never set OpenPersona `sourceIdentity` for statistical / synthetic corpora. +3. Sensitive or weakly evidenced attributes go into `constraints.sensitiveFlags` or are omitted — not silently asserted as facts. +4. `toSeed` must populate `gaps` for anything the shared mapper cannot invent (at least `personaName`, `slug`, `role` when absent). +5. Provenance is written by `scripts/write-provenance.js` to `soul/seed-provenance.json` after pack generation (not as a `persona.json` root key). + +## Adding a provider + +1. Create `providers//provider.js` exporting `{ capabilities, search, fetch, toSeed }`. +2. Document field mapping in `providers//mapping.md`. +3. Register the id in `scripts/search.js` `PROVIDERS` map. +4. Add a small fixture so offline tests pass without network. diff --git a/skills/persona-seed/schemas/intent.schema.json b/skills/persona-seed/schemas/intent.schema.json new file mode 100644 index 0000000..047c599 --- /dev/null +++ b/skills/persona-seed/schemas/intent.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openpersona.org/schemas/persona-seed/intent.schema.json", + "title": "PersonaSeedIntent", + "description": "Normalized user intent for searching persona seed providers.", + "type": "object", + "additionalProperties": false, + "properties": { + "roleHint": { + "type": "string", + "description": "OpenPersona role hint (companion, mentor, assistant, …). Not used as a hard MatrAIx filter." + }, + "domain": { + "type": "array", + "items": { "type": "string" }, + "description": "Domain / occupation keywords to match against seed capabilities." + }, + "traits": { + "type": "array", + "items": { "type": "string" }, + "description": "Desired personality / value keywords." + }, + "locale": { + "type": "array", + "items": { "type": "string" }, + "description": "Language / locale codes or names (e.g. en, zh, Spanish)." + }, + "region": { + "type": "array", + "items": { "type": "string" }, + "description": "Preferred regions (soft filter)." + }, + "ageBracket": { + "type": "array", + "items": { "type": "string" } + }, + "exclude": { + "type": "array", + "items": { "type": "string" }, + "description": "Keywords or constraint flags to avoid." + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 20, + "default": 5 + }, + "provider": { + "type": "string", + "description": "Provider id. Default: matraix-persona-1m when only one is installed." + }, + "query": { + "type": "string", + "description": "Free-text user request; agent may leave structured fields empty and rely on rerank." + } + } +} diff --git a/skills/persona-seed/schemas/seed-profile.schema.json b/skills/persona-seed/schemas/seed-profile.schema.json new file mode 100644 index 0000000..58ada45 --- /dev/null +++ b/skills/persona-seed/schemas/seed-profile.schema.json @@ -0,0 +1,106 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openpersona.org/schemas/persona-seed/seed-profile.schema.json", + "title": "SeedProfile", + "description": "Provider-agnostic intermediate persona seed. Mapped to persona.json by the shared mapper; never fed directly to the generator.", + "type": "object", + "required": ["schemaVersion", "provenance", "identity", "character", "gaps"], + "additionalProperties": false, + "properties": { + "schemaVersion": { + "type": "string", + "const": "0.1.0" + }, + "provenance": { + "type": "object", + "required": ["provider", "recordId"], + "additionalProperties": false, + "properties": { + "provider": { "type": "string" }, + "recordId": { "type": "string" }, + "dataset": { "type": "string" }, + "corpusMode": { + "type": "string", + "enum": ["fixture", "external"], + "description": "fixture = offline sample rows; external = MATRAIX_CORPUS_PATH (or future HF-backed index)" + }, + "licenseNotes": { "type": "string" }, + "groundingType": { + "type": "string", + "enum": ["synthetic", "human_grounded", "unknown"] + }, + "retrievedAt": { "type": "string" } + } + }, + "identity": { + "type": "object", + "required": ["summary"], + "additionalProperties": false, + "properties": { + "suggestedName": { "type": ["string", "null"] }, + "suggestedSlug": { "type": ["string", "null"] }, + "summary": { "type": "string", "minLength": 1 }, + "locale": { "type": "array", "items": { "type": "string" } }, + "region": { "type": ["string", "null"] }, + "ageBracket": { "type": ["string", "null"] }, + "occupation": { "type": ["string", "null"] }, + "education": { "type": ["string", "null"] } + } + }, + "character": { + "type": "object", + "additionalProperties": false, + "properties": { + "traits": { "type": "array", "items": { "type": "string" } }, + "values": { "type": "array", "items": { "type": "string" } }, + "motivations": { "type": "array", "items": { "type": "string" } }, + "speakingHints": { "type": "array", "items": { "type": "string" } }, + "interests": { "type": "array", "items": { "type": "string" } }, + "riskTolerance": { "type": ["string", "null"] }, + "formalityBaseline": { "type": ["number", "null"] } + } + }, + "capabilities": { + "type": "object", + "additionalProperties": false, + "properties": { + "domains": { "type": "array", "items": { "type": "string" } }, + "skills": { "type": "array", "items": { "type": "string" } }, + "tools": { "type": "array", "items": { "type": "string" } } + } + }, + "constraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "hardExclusions": { "type": "array", "items": { "type": "string" } }, + "sensitiveFlags": { "type": "array", "items": { "type": "string" } }, + "suggestedImmutableTraits": { "type": "array", "items": { "type": "string" } } + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "properties": { + "attributeHighlights": { + "type": "array", + "items": { + "type": "object", + "required": ["field", "value"], + "additionalProperties": false, + "properties": { + "field": { "type": "string" }, + "value": { "type": "string" } + } + } + }, + "rawDescriptions": { "type": "array", "items": { "type": "string" } } + } + }, + "gaps": { + "type": "array", + "items": { "type": "string" }, + "description": "Fields the agent/user must still fill before create (e.g. personaName, role, boundaries)." + } + } +} diff --git a/skills/persona-seed/scripts/map-seed-to-persona.js b/skills/persona-seed/scripts/map-seed-to-persona.js new file mode 100644 index 0000000..7149c79 --- /dev/null +++ b/skills/persona-seed/scripts/map-seed-to-persona.js @@ -0,0 +1,224 @@ +#!/usr/bin/env node +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +/** + * Shared SeedProfile → persona.json mapper. + * Fills structural defaults from a base-shaped skeleton. + * Does not invent a display name when overrides omit it — caller must pass identity overrides. + */ + +/** Root keys safe to merge from overrides.extra (never soul/body/evolution). */ +const EXTRA_ALLOWED_ROOT_KEYS = new Set([ + 'faculties', + 'skills', + 'economy', + 'vitality', + 'social', + 'rhythm', + 'memory', + 'additionalAllowedTools', + 'version', + 'author', + 'packType', +]); + +function slugify(name) { + const ascii = String(name || '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 48); + if (ascii) return ascii; + const hash = crypto.createHash('sha256').update(String(name || '')).digest('hex').slice(0, 8); + return `persona-${hash}`; +} + +function buildSpeakingStyle(seed, override) { + if (override) return override; + const hints = seed.character?.speakingHints || []; + if (!hints.length) { + return 'Clear and natural. Adapts to the user while staying consistent with the seeded personality.'; + } + return `Follow these speaking tendencies: ${hints.join('; ')}. Stay consistent with the seeded traits without caricature.`; +} + +function buildBoundaries(seed, override) { + if (override) return override; + const flags = seed.constraints?.sensitiveFlags || []; + const lines = [ + 'Follow the OpenPersona Universal Constitution (Safety > Honesty > Helpfulness).', + 'Honest about being an AI. Do not claim to be a real person represented by the seed corpus.', + ]; + if (flags.includes('healthcare_domain')) { + lines.push( + 'No clinical diagnosis or prescribing. Encourage licensed professionals for medical decisions.' + ); + } + return lines.join(' '); +} + +function buildBackground(seed) { + const parts = [seed.identity?.summary].filter(Boolean); + const occ = seed.identity?.occupation; + const region = seed.identity?.region; + if (occ || region) { + parts.push(`Seeded context: ${[occ, region].filter(Boolean).join(' · ')}.`); + } + const interests = seed.character?.interests || []; + if (interests.length) parts.push(`Interests include ${interests.slice(0, 6).join(', ')}.`); + return parts.join('\n\n'); +} + +function applyExtra(persona, extra) { + if (!extra || typeof extra !== 'object' || Array.isArray(extra)) { + throw new Error('overrides.extra must be a plain object when provided'); + } + for (const key of Object.keys(extra)) { + if (!EXTRA_ALLOWED_ROOT_KEYS.has(key)) { + throw new Error( + `overrides.extra has disallowed root key "${key}". ` + + `Allowed: ${[...EXTRA_ALLOWED_ROOT_KEYS].join(', ')}. ` + + 'Do not override soul/body/evolution via extra.' + ); + } + persona[key] = extra[key]; + } +} + +/** + * @param {object} seed SeedProfile + * @param {object} overrides + * @param {string} overrides.personaName required + * @param {string} [overrides.slug] + * @param {string} [overrides.role] + * @param {string} [overrides.bio] + * @param {string} [overrides.personality] + * @param {string} [overrides.speakingStyle] + * @param {string} [overrides.boundaries] + * @param {object} [overrides.extra] whitelisted root keys only + */ +function mapSeedToPersona(seed, overrides = {}) { + if (!overrides.personaName) { + throw new Error('mapSeedToPersona requires overrides.personaName (fill SeedProfile gaps first)'); + } + + const slug = overrides.slug || slugify(overrides.personaName); + const role = overrides.role || 'assistant'; + const traits = seed.character?.traits || []; + const personality = + overrides.personality || + (traits.length ? traits.join(', ') : 'adaptive, attentive, honest'); + + const immutable = Array.from( + new Set([...(seed.constraints?.suggestedImmutableTraits || []), 'honest', 'helpful']) + ); + + const formality = + typeof seed.character?.formalityBaseline === 'number' + ? seed.character.formalityBaseline + : 0; + const minFormality = Math.max(-10, Math.min(formality - 3, 9)); + const maxFormality = Math.min(10, Math.max(formality + 3, minFormality + 1)); + + const persona = { + soul: { + identity: { + personaName: overrides.personaName, + slug, + role, + bio: overrides.bio || seed.identity?.summary || `${overrides.personaName} persona`, + }, + character: { + personality, + speakingStyle: buildSpeakingStyle(seed, overrides.speakingStyle), + boundaries: buildBoundaries(seed, overrides.boundaries), + background: buildBackground(seed), + }, + }, + body: { + runtime: { + framework: 'openclaw', + }, + }, + faculties: [{ name: 'memory' }], + skills: [], + evolution: { + instance: { + enabled: true, + relationshipProgression: true, + moodTracking: true, + traitEmergence: true, + speakingStyleDrift: true, + interestDiscovery: true, + boundaries: { + immutableTraits: immutable, + minFormality, + maxFormality, + }, + }, + }, + social: { + acn: { enabled: true, gateway: 'https://acn-production.up.railway.app' }, + onchain: { chain: 'base' }, + a2a: { enabled: true, protocol: '0.3.0' }, + }, + rhythm: { + heartbeat: { enabled: false }, + }, + version: '0.1.0', + author: overrides.author || 'persona-seed', + }; + + if (overrides.extra !== undefined) { + applyExtra(persona, overrides.extra); + } + + return persona; +} + +function main() { + const args = process.argv.slice(2); + let seedPath; + let overrides = {}; + let outPath; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--seed') seedPath = args[++i]; + else if (args[i] === '--overrides') { + const raw = args[++i]; + overrides = JSON.parse(raw.startsWith('{') ? raw : fs.readFileSync(raw, 'utf8')); + } else if (args[i] === '--out') outPath = args[++i]; + } + + if (!seedPath) { + process.stderr.write( + 'Usage: map-seed-to-persona.js --seed --overrides [--out persona.json]\n' + ); + process.exit(1); + } + + const seed = JSON.parse(fs.readFileSync(seedPath, 'utf8')); + const persona = mapSeedToPersona(seed, overrides); + const text = `${JSON.stringify(persona, null, 2)}\n`; + if (outPath) { + fs.mkdirSync(path.dirname(path.resolve(outPath)), { recursive: true }); + fs.writeFileSync(outPath, text); + } else { + process.stdout.write(text); + } +} + +if (require.main === module) { + try { + main(); + } catch (err) { + process.stderr.write(`${err.message}\n`); + process.exit(1); + } +} + +module.exports = { mapSeedToPersona, slugify, EXTRA_ALLOWED_ROOT_KEYS }; diff --git a/skills/persona-seed/scripts/prepare-corpus.js b/skills/persona-seed/scripts/prepare-corpus.js new file mode 100644 index 0000000..2f11ad7 --- /dev/null +++ b/skills/persona-seed/scripts/prepare-corpus.js @@ -0,0 +1,164 @@ +#!/usr/bin/env node +'use strict'; + +/** + * Validate / normalize a decoded persona corpus for MATRAIX_CORPUS_PATH. + * + * Expected record shape (same as fixtures/sample-corpus.json): + * { + * "id": "string", + * "groundingType": "synthetic" | "human_grounded" | "unknown", + * "description": "string", + * "attributes": { ... string | string[] values ... } + * } + * + * Hugging Face release (MatrAIx2026/MatrAIx_Persona_1M) ships packed Parquet — + * decode offline (Python + pyarrow per dataset card), then point this tool at + * the resulting JSON array / JSONL. + * + * Usage: + * node prepare-corpus.js --validate /path/to/corpus.json + * node prepare-corpus.js --validate /path/to/corpus.jsonl --limit 100 + * node prepare-corpus.js --print-fixture-schema + */ + +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); + +const FIXTURE = path.join( + __dirname, + '..', + 'providers', + 'matraix-persona-1m', + 'fixtures', + 'sample-corpus.json' +); + +function parseArgs(argv) { + const out = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--validate') out.validate = argv[++i]; + else if (a === '--limit') out.limit = Number(argv[++i]); + else if (a === '--print-fixture-schema') out.printSchema = true; + else if (a === '--help' || a === '-h') out.help = true; + } + return out; +} + +function validateRecord(rec, index) { + const errors = []; + if (!rec || typeof rec !== 'object') { + return [`[${index}] not an object`]; + } + if (!rec.id || typeof rec.id !== 'string') errors.push(`[${index}] missing string id`); + if (!rec.attributes || typeof rec.attributes !== 'object') { + errors.push(`[${index}] missing attributes object`); + } + if (rec.description != null && typeof rec.description !== 'string') { + errors.push(`[${index}] description must be string when present`); + } + return errors; +} + +async function loadRecords(filePath, limit) { + if (filePath.endsWith('.jsonl')) { + const records = []; + const rl = readline.createInterface({ + input: fs.createReadStream(filePath, { encoding: 'utf8' }), + crlfDelay: Infinity, + }); + let i = 0; + for await (const line of rl) { + const trimmed = line.trim(); + if (!trimmed) continue; + records.push(JSON.parse(trimmed)); + i++; + if (limit && i >= limit) break; + } + return records; + } + + const data = JSON.parse(fs.readFileSync(filePath, 'utf8')); + if (!Array.isArray(data)) { + throw new Error('JSON corpus must be an array of records'); + } + return limit ? data.slice(0, limit) : data; +} + +async function validate(filePath, limit) { + const records = await loadRecords(filePath, limit); + const errors = []; + const ids = new Set(); + for (let i = 0; i < records.length; i++) { + errors.push(...validateRecord(records[i], i)); + if (records[i]?.id) { + if (ids.has(records[i].id)) errors.push(`[${i}] duplicate id ${records[i].id}`); + ids.add(records[i].id); + } + } + return { + ok: errors.length === 0, + count: records.length, + errors: errors.slice(0, 50), + truncatedErrors: errors.length > 50, + }; +} + +function printHelp() { + process.stdout.write(`prepare-corpus.js — validate decoded MatrAIx-shaped JSON for persona-seed + +Decode the HF parquet yourself (see dataset card), emit JSON/JSONL in the fixture shape, then: + + export MATRAIX_CORPUS_PATH=/path/to/corpus.json + node scripts/search.js --intent '{"domain":["software"]}' + +Commands: + --validate [--limit N] + --print-fixture-schema +`); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help || (!args.validate && !args.printSchema)) { + printHelp(); + process.exit(args.help ? 0 : 1); + } + + if (args.printSchema) { + const fixture = JSON.parse(fs.readFileSync(FIXTURE, 'utf8')); + process.stdout.write( + `${JSON.stringify( + { + recordShape: { + id: 'string', + groundingType: 'synthetic|human_grounded|unknown', + description: 'string', + attributes: 'object of string | string[]', + }, + example: fixture[0], + downloadHint: + 'huggingface-cli download MatrAIx2026/MatrAIx_Persona_1M --repo-type dataset --local-dir ./matraix-1m', + }, + null, + 2 + )}\n` + ); + return; + } + + const result = await validate(args.validate, args.limit); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + if (!result.ok) process.exit(1); +} + +if (require.main === module) { + main().catch((err) => { + process.stderr.write(`${err.message}\n`); + process.exit(1); + }); +} + +module.exports = { validateRecord, validate, loadRecords }; diff --git a/skills/persona-seed/scripts/run-pipeline.js b/skills/persona-seed/scripts/run-pipeline.js new file mode 100644 index 0000000..cf724a3 --- /dev/null +++ b/skills/persona-seed/scripts/run-pipeline.js @@ -0,0 +1,133 @@ +#!/usr/bin/env node +'use strict'; + +/** + * End-to-end: search → toSeed → map → generate → write provenance. + * Usage: + * node run-pipeline.js --intent '{"domain":["software"],"traits":["precise"]}' \ + * --name "Seeded Nova" --slug seeded-nova --role assistant --out /tmp/op-seed-out + * + * Or pick a record: + * node run-pipeline.js --id fixture-001 --name "Seeded Nova" --slug seeded-nova --out /tmp/op-seed-out + */ + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +const { resolveProvider } = require('./search'); +const { mapSeedToPersona } = require('./map-seed-to-persona'); +const { writeProvenance } = require('./write-provenance'); + +function parseArgs(argv) { + const out = { _: [] }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if ( + a === '--intent' || + a === '--id' || + a === '--provider' || + a === '--name' || + a === '--slug' || + a === '--role' || + a === '--out' || + a === '--overrides' + ) { + out[a.slice(2)] = argv[++i]; + } else if (a === '--dry-map') { + out.dryMap = true; + } else { + out._.push(a); + } + } + return out; +} + +async function run(args) { + const provider = resolveProvider(args.provider); + let recordId = args.id; + + if (!recordId) { + let intent = {}; + if (args.intent) { + intent = JSON.parse( + args.intent.startsWith('{') ? args.intent : fs.readFileSync(args.intent, 'utf8') + ); + } else if (args._[0]) { + intent = { query: args._.join(' '), limit: 5 }; + } else { + intent = { limit: 5 }; + } + intent.limit = intent.limit || 5; + const hits = provider.search(intent); + if (!hits.length) { + throw new Error('search returned no candidates; broaden intent or pass --id'); + } + recordId = hits[0].id; + process.stderr.write( + `[persona-seed] search top=${recordId} score=${hits[0].score} (${hits.length} hit(s))\n` + ); + } + + if (!args.name) { + throw new Error('--name is required'); + } + + const raw = provider.fetch(recordId); + const seed = provider.toSeed(raw); + + let overrides = { + personaName: args.name, + slug: args.slug, + role: args.role || 'assistant', + }; + if (args.overrides) { + const extra = JSON.parse( + args.overrides.startsWith('{') ? args.overrides : fs.readFileSync(args.overrides, 'utf8') + ); + overrides = { ...overrides, ...extra, personaName: args.name || extra.personaName }; + if (args.slug) overrides.slug = args.slug; + if (args.role) overrides.role = args.role; + } + + const persona = mapSeedToPersona(seed, overrides); + const outRoot = path.resolve(args.out || path.join(os.tmpdir(), 'persona-seed-pipeline')); + fs.mkdirSync(outRoot, { recursive: true }); + + const seedPath = path.join(outRoot, 'seed-profile.json'); + const personaPath = path.join(outRoot, 'persona.json'); + fs.writeFileSync(seedPath, `${JSON.stringify(seed, null, 2)}\n`); + fs.writeFileSync(personaPath, `${JSON.stringify(persona, null, 2)}\n`); + + if (args.dryMap) { + return { seedPath, personaPath, recordId, dryMap: true }; + } + + // Lazy-require generator so --dry-map works without full framework load side effects in odd envs + const { generate } = require(path.resolve(__dirname, '../../../lib/generator')); + const { skillDir } = await generate(persona, outRoot); + const provenancePath = writeProvenance(skillDir, seed); + + return { + recordId, + seedPath, + personaPath, + skillDir, + provenancePath, + corpusMode: seed.provenance.corpusMode, + slug: persona.soul.identity.slug, + }; +} + +if (require.main === module) { + run(parseArgs(process.argv.slice(2))) + .then((result) => { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + }) + .catch((err) => { + process.stderr.write(`${err.message}\n`); + process.exit(1); + }); +} + +module.exports = { run }; diff --git a/skills/persona-seed/scripts/search.js b/skills/persona-seed/scripts/search.js new file mode 100644 index 0000000..f0ea0eb --- /dev/null +++ b/skills/persona-seed/scripts/search.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node +'use strict'; + +const matraix = require('../providers/matraix-persona-1m/provider'); + +const PROVIDERS = { + [matraix.PROVIDER_ID]: matraix, +}; + +function resolveProvider(id) { + const key = id || matraix.PROVIDER_ID; + const provider = PROVIDERS[key]; + if (!provider) { + throw new Error( + `Unknown provider "${key}". Registered: ${Object.keys(PROVIDERS).join(', ')}` + ); + } + return provider; +} + +function parseArgs(argv) { + const out = { _: [] }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--intent' || a === '--id' || a === '--provider' || a === '--out') { + out[a.slice(2)] = argv[++i]; + } else if (a === '--fetch') { + out.fetch = true; + } else if (a === '--to-seed') { + out.toSeed = true; + } else if (a === '--capabilities') { + out.capabilities = true; + } else { + out._.push(a); + } + } + return out; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const provider = resolveProvider(args.provider); + + if (args.capabilities) { + process.stdout.write(`${JSON.stringify(provider.capabilities(), null, 2)}\n`); + return; + } + + if (args.fetch || args.toSeed) { + const id = args.id || args._[0]; + if (!id) throw new Error('fetch/to-seed requires --id '); + const raw = provider.fetch(id); + const payload = args.toSeed ? provider.toSeed(raw) : raw; + process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`); + return; + } + + let intent = {}; + if (args.intent) { + intent = JSON.parse(args.intent.startsWith('{') ? args.intent : require('fs').readFileSync(args.intent, 'utf8')); + } else if (args._[0]) { + intent = { query: args._.join(' '), limit: 5 }; + } + + if (args.provider) intent.provider = args.provider; + const results = provider.search(intent); + process.stdout.write(`${JSON.stringify(results, null, 2)}\n`); +} + +if (require.main === module) { + try { + main(); + } catch (err) { + process.stderr.write(`${err.message}\n`); + process.exit(1); + } +} + +module.exports = { PROVIDERS, resolveProvider }; diff --git a/skills/persona-seed/scripts/write-provenance.js b/skills/persona-seed/scripts/write-provenance.js new file mode 100644 index 0000000..538d773 --- /dev/null +++ b/skills/persona-seed/scripts/write-provenance.js @@ -0,0 +1,59 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +/** + * Write seed provenance beside the generated pack (not into persona.json root). + * Target: /soul/seed-provenance.json + */ + +function writeProvenance(packDir, seedOrProvenance) { + const provenance = seedOrProvenance.provenance || seedOrProvenance; + if (!provenance || !provenance.provider || !provenance.recordId) { + throw new Error('writeProvenance requires provenance.provider and provenance.recordId'); + } + + const soulDir = path.join(packDir, 'soul'); + fs.mkdirSync(soulDir, { recursive: true }); + const outPath = path.join(soulDir, 'seed-provenance.json'); + const payload = { + schemaVersion: '0.1.0', + ...provenance, + writtenAt: new Date().toISOString(), + }; + fs.writeFileSync(outPath, `${JSON.stringify(payload, null, 2)}\n`); + return outPath; +} + +function main() { + const args = process.argv.slice(2); + let packDir; + let seedPath; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--pack') packDir = args[++i]; + else if (args[i] === '--seed') seedPath = args[++i]; + } + + if (!packDir || !seedPath) { + process.stderr.write('Usage: write-provenance.js --pack --seed \n'); + process.exit(1); + } + + const seed = JSON.parse(fs.readFileSync(seedPath, 'utf8')); + const outPath = writeProvenance(packDir, seed); + process.stdout.write(`${outPath}\n`); +} + +if (require.main === module) { + try { + main(); + } catch (err) { + process.stderr.write(`${err.message}\n`); + process.exit(1); + } +} + +module.exports = { writeProvenance }; diff --git a/skills/persona-seed/tests/persona-seed.test.js b/skills/persona-seed/tests/persona-seed.test.js new file mode 100644 index 0000000..7788338 --- /dev/null +++ b/skills/persona-seed/tests/persona-seed.test.js @@ -0,0 +1,180 @@ +'use strict'; + +const { describe, it, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const matraix = require('../providers/matraix-persona-1m/provider'); +const { mapSeedToPersona, slugify } = require('../scripts/map-seed-to-persona'); +const { writeProvenance } = require('../scripts/write-provenance'); +const { run: runPipeline } = require('../scripts/run-pipeline'); +const { validate } = require('../scripts/prepare-corpus'); +const { generate } = require('../../../lib/generator'); + +describe('persona-seed / matraix-persona-1m', () => { + it('capabilities exposes fixture mode by default', () => { + const caps = matraix.capabilities(); + assert.equal(caps.id, 'matraix-persona-1m'); + assert.equal(caps.corpusMode, 'fixture'); + assert.match(caps.dataset, /fixtures\/sample-corpus/); + }); + + it('search filters by domain and traits', () => { + const hits = matraix.search({ + domain: ['software'], + traits: ['precise'], + limit: 5, + }); + assert.ok(hits.length >= 1); + assert.ok(hits.some((h) => h.id === 'fixture-001')); + assert.ok(hits[0].score > 0); + }); + + it('search returns empty when positive filters match nothing', () => { + const hits = matraix.search({ domain: ['astrobiology'], limit: 5 }); + assert.deepEqual(hits, []); + }); + + it('search hard-excludes matches and does not fall back to full corpus', () => { + const hits = matraix.search({ exclude: ['software'], limit: 10 }); + assert.ok(hits.length >= 1); + assert.ok(!hits.some((h) => h.id === 'fixture-001')); + assert.ok(!hits.some((h) => h.id === 'fixture-005')); + }); + + it('toSeed fills gaps and fixture provenance', () => { + const seed = matraix.toSeed(matraix.fetch('fixture-001')); + assert.equal(seed.schemaVersion, '0.1.0'); + assert.equal(seed.provenance.provider, 'matraix-persona-1m'); + assert.equal(seed.provenance.recordId, 'fixture-001'); + assert.equal(seed.provenance.corpusMode, 'fixture'); + assert.match(seed.provenance.dataset, /fixtures\/sample-corpus/); + assert.ok(seed.identity.summary.length > 0); + assert.ok(seed.gaps.includes('personaName')); + assert.ok(seed.gaps.includes('role')); + }); + + it('healthcare seed flags sensitive domain', () => { + const seed = matraix.toSeed(matraix.fetch('fixture-003')); + assert.ok(seed.constraints.sensitiveFlags.includes('healthcare_domain')); + }); +}); + +describe('persona-seed / mapper + provenance', () => { + let tmp; + + before(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'persona-seed-')); + }); + + after(() => { + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it('mapSeedToPersona requires personaName', () => { + const seed = matraix.toSeed(matraix.fetch('fixture-002')); + assert.throws(() => mapSeedToPersona(seed, {}), /personaName/); + }); + + it('mapSeedToPersona emits required soul fields', () => { + const seed = matraix.toSeed(matraix.fetch('fixture-002')); + const persona = mapSeedToPersona(seed, { + personaName: 'Coach Ada', + slug: 'coach-ada', + role: 'mentor', + }); + assert.equal(persona.soul.identity.personaName, 'Coach Ada'); + assert.equal(persona.soul.identity.slug, 'coach-ada'); + assert.equal(persona.soul.identity.role, 'mentor'); + assert.ok(persona.soul.character.personality); + assert.ok(persona.soul.character.speakingStyle); + assert.ok(persona.evolution.instance.enabled); + }); + + it('slugify keeps ascii and hashes non-latin names', () => { + assert.equal(slugify('Coach Ada'), 'coach-ada'); + const zh = slugify('导师小明'); + assert.match(zh, /^persona-[a-f0-9]{8}$/); + assert.equal(slugify('导师小明'), zh); + }); + + it('rejects disallowed overrides.extra root keys', () => { + const seed = matraix.toSeed(matraix.fetch('fixture-002')); + assert.throws( + () => + mapSeedToPersona(seed, { + personaName: 'X', + extra: { soul: { identity: {} } }, + }), + /disallowed root key/ + ); + }); + + it('writeProvenance writes soul/seed-provenance.json', () => { + const seed = matraix.toSeed(matraix.fetch('fixture-001')); + const packDir = path.join(tmp, 'pack'); + const out = writeProvenance(packDir, seed); + assert.equal(path.basename(out), 'seed-provenance.json'); + const written = JSON.parse(fs.readFileSync(out, 'utf8')); + assert.equal(written.provider, 'matraix-persona-1m'); + assert.equal(written.recordId, 'fixture-001'); + assert.equal(written.corpusMode, 'fixture'); + }); + + it('mapped persona passes generate gate', async () => { + const seed = matraix.toSeed(matraix.fetch('fixture-001')); + const persona = mapSeedToPersona(seed, { + personaName: 'Seeded Gate', + slug: 'seeded-gate-e2e', + role: 'assistant', + }); + const outDir = path.join(tmp, 'gen'); + fs.mkdirSync(outDir, { recursive: true }); + const { skillDir } = await generate(persona, outDir); + assert.ok(fs.existsSync(path.join(skillDir, 'SKILL.md'))); + assert.ok(fs.existsSync(path.join(skillDir, 'persona.json'))); + assert.ok(fs.existsSync(path.join(skillDir, 'soul', 'injection.md'))); + writeProvenance(skillDir, seed); + const prov = JSON.parse( + fs.readFileSync(path.join(skillDir, 'soul', 'seed-provenance.json'), 'utf8') + ); + assert.equal(prov.recordId, 'fixture-001'); + assert.equal(prov.corpusMode, 'fixture'); + }); +}); + +describe('persona-seed / run-pipeline', () => { + it('search → seed → map → generate → provenance', async () => { + const out = path.join(os.tmpdir(), `persona-seed-pipeline-${Date.now()}`); + const result = await runPipeline({ + intent: JSON.stringify({ domain: ['software'], traits: ['precise'], limit: 3 }), + name: 'Pipeline Nova', + slug: 'pipeline-nova-e2e', + role: 'assistant', + out, + }); + assert.equal(result.recordId, 'fixture-001'); + assert.ok(fs.existsSync(result.skillDir)); + assert.ok(fs.existsSync(result.provenancePath)); + assert.ok(fs.existsSync(path.join(result.skillDir, 'SKILL.md'))); + fs.rmSync(out, { recursive: true, force: true }); + }); +}); + +describe('persona-seed / prepare-corpus', () => { + it('validates the offline fixture corpus', async () => { + const fixture = path.join( + __dirname, + '..', + 'providers', + 'matraix-persona-1m', + 'fixtures', + 'sample-corpus.json' + ); + const result = await validate(fixture); + assert.equal(result.ok, true); + assert.equal(result.count, 5); + }); +}); diff --git a/tests/persona-seed.test.js b/tests/persona-seed.test.js new file mode 100644 index 0000000..c7657a9 --- /dev/null +++ b/tests/persona-seed.test.js @@ -0,0 +1,4 @@ +'use strict'; + +// Thin entry so `npm test` discovers the skill-local suite. +require('../skills/persona-seed/tests/persona-seed.test.js'); From d8f6a46adc0be6f05ec69bb3bd7cb9b03041f2b8 Mon Sep 17 00:00:00 2001 From: NeilJo-GY <43027886+NeilJo-GY@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:03:12 +0800 Subject: [PATCH 2/4] Add MatrAIx parquet decode path and JSONL corpus support Provide decode_parquet.py / decode-matraix.sh to turn HF packed shards into fixture-shaped JSON for MATRAIX_CORPUS_PATH, and accept .jsonl corpora in the provider for larger offline indexes. Co-authored-by: Cursor --- skills/persona-seed/SKILL.md | 11 +- .../providers/matraix-persona-1m/mapping.md | 2 +- .../providers/matraix-persona-1m/provider.js | 24 +- .../scripts/decode_parquet.py | 301 ++++++++++++++++++ skills/persona-seed/scripts/decode-matraix.sh | 62 ++++ .../persona-seed/tests/persona-seed.test.js | 30 ++ 6 files changed, 425 insertions(+), 5 deletions(-) create mode 100755 skills/persona-seed/providers/matraix-persona-1m/scripts/decode_parquet.py create mode 100755 skills/persona-seed/scripts/decode-matraix.sh diff --git a/skills/persona-seed/SKILL.md b/skills/persona-seed/SKILL.md index a0160ac..bf7befe 100644 --- a/skills/persona-seed/SKILL.md +++ b/skills/persona-seed/SKILL.md @@ -58,7 +58,16 @@ Set `SEED_DIR` to this skill directory (path containing `SKILL.md`). Default corpus is the offline **fixture** (5 rows) under `providers/matraix-persona-1m/fixtures/`. `capabilities()` / provenance report `corpusMode: "fixture"` and a fixture dataset id — not the Hugging Face 1M release. -To use a larger decoded corpus: decode HF Parquet offline → JSON/JSONL in the fixture record shape → `prepare-corpus.js --validate` → `export MATRAIX_CORPUS_PATH=…`. +To use a larger decoded corpus: + +```bash +# optional download + decode first N rows (needs python3 + pyarrow [+ huggingface-cli]) +bash ${SEED_DIR}/scripts/decode-matraix.sh --download --limit 1000 +export MATRAIX_CORPUS_PATH=./matraix-1m.decoded.json # or .jsonl +node ${SEED_DIR}/scripts/prepare-corpus.js --validate "$MATRAIX_CORPUS_PATH" --limit 20 +``` + +Decoder: `providers/matraix-persona-1m/scripts/decode_parquet.py` Empty filter results return `[]` (no silent unfiltered fallback). Re-check intent if search is empty. diff --git a/skills/persona-seed/providers/matraix-persona-1m/mapping.md b/skills/persona-seed/providers/matraix-persona-1m/mapping.md index 3d87f3c..d186fc8 100644 --- a/skills/persona-seed/providers/matraix-persona-1m/mapping.md +++ b/skills/persona-seed/providers/matraix-persona-1m/mapping.md @@ -30,7 +30,7 @@ MatrAIx Persona 1M stores ~1,290 categorical attributes (packed in the HF releas - Do **not** set OpenPersona `sourceIdentity` (archetype / synthetic seed). - Do **not** invent `personaName` in the adapter; leave for agent + user. -- Full HF parquet decode is v2; v1 uses fixtures or a decoded JSON corpus path via `MATRAIX_CORPUS_PATH`. +- Packed HF Parquet → fixture-shaped JSON/JSONL via `scripts/decode_parquet.py` (or `skills/persona-seed/scripts/decode-matraix.sh`). ## Dataset / corpusMode diff --git a/skills/persona-seed/providers/matraix-persona-1m/provider.js b/skills/persona-seed/providers/matraix-persona-1m/provider.js index 4db5d4e..a1a4ce7 100644 --- a/skills/persona-seed/providers/matraix-persona-1m/provider.js +++ b/skills/persona-seed/providers/matraix-persona-1m/provider.js @@ -32,6 +32,17 @@ function resetCorpusCache() { corpusCache = null; } +function loadJsonl(filePath) { + const text = fs.readFileSync(filePath, 'utf8'); + const rows = []; + for (const line of text.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed) continue; + rows.push(JSON.parse(trimmed)); + } + return rows; +} + function loadCorpus() { const resolved = corpusPath(); const stat = fs.statSync(resolved); @@ -43,10 +54,17 @@ function loadCorpus() { return corpusCache.data; } - const raw = fs.readFileSync(resolved, 'utf8'); - const data = JSON.parse(raw); + let data; + if (resolved.endsWith('.jsonl')) { + data = loadJsonl(resolved); + } else { + const raw = fs.readFileSync(resolved, 'utf8'); + data = JSON.parse(raw); + } if (!Array.isArray(data)) { - throw new Error(`matraix-persona-1m: corpus must be a JSON array: ${resolved}`); + throw new Error( + `matraix-persona-1m: corpus must be a JSON array or .jsonl: ${resolved}` + ); } corpusCache = { path: resolved, mtimeMs: stat.mtimeMs, data }; return data; diff --git a/skills/persona-seed/providers/matraix-persona-1m/scripts/decode_parquet.py b/skills/persona-seed/providers/matraix-persona-1m/scripts/decode_parquet.py new file mode 100755 index 0000000..ccfed58 --- /dev/null +++ b/skills/persona-seed/providers/matraix-persona-1m/scripts/decode_parquet.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Decode MatrAIx Persona 1M packed Parquet shards into persona-seed JSONL. + +Requires: Python 3 + pyarrow + +Example: + huggingface-cli download MatrAIx2026/MatrAIx_Persona_1M \\ + --repo-type dataset --local-dir ./matraix-1m + + python3 decode_parquet.py \\ + --dataset-dir ./matraix-1m \\ + --out ./matraix-1m.decoded.jsonl \\ + --limit 1000 + +Then: + export MATRAIX_CORPUS_PATH=./matraix-1m.decoded.jsonl + # search.js / prepare-corpus.js accept JSON arrays; for JSONL use prepare-corpus + # or convert: python3 -c 'import json; ...' → .json array for the provider. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +# High-signal fields → fixture attribute keys (identity preserved when same). +FIELD_MAP = { + "region": "region", + "age_bracket": "age_bracket", + "primary_language": "primary_language", + "highest_education": "highest_education", + "occupation": "occupation", + "occupation_domain": "occupation_domain", + "industry": "occupation_domain", + "risk_tolerance": "risk_tolerance", + "communication_formality": "communication_formality", +} + +# Collect into arrays when present under these source ids / prefixes. +LIST_FIELDS = { + "personality_traits": ("personality_traits", "big_five", "trait_"), + "values": ("values", "value_"), + "motivations": ("motivations", "motivation_"), + "interests": ("interests", "hobby", "interest_"), + "speaking_style_hints": ("speaking_style", "communication_style", "interaction_"), + "tools": ("tools", "programming_language", "software_tool"), +} + + +def load_schema(dataset_dir: Path) -> list[dict[str, Any]]: + schema_path = dataset_dir / "persona_codes.schema.json" + if not schema_path.exists(): + # Some releases nest under release/ + alt = dataset_dir / "release" / "persona_codes.schema.json" + schema_path = alt if alt.exists() else schema_path + if not schema_path.exists(): + raise SystemExit(f"persona_codes.schema.json not found under {dataset_dir}") + data = json.loads(schema_path.read_text(encoding="utf-8")) + cols = data.get("columns") + if not isinstance(cols, list): + raise SystemExit("persona_codes.schema.json missing columns[]") + return cols + + +def decode_attributes( + attributes: bytes | None, + null_bitmap: bytes | None, + schema: list[dict[str, Any]], + overrides: dict[str, Any] | None, +) -> dict[str, str]: + out: dict[str, str] = {} + if attributes is None: + return out + if isinstance(attributes, memoryview): + attributes = attributes.tobytes() + elif not isinstance(attributes, (bytes, bytearray)): + attributes = bytes(attributes) + + nb = None + if null_bitmap is not None: + if isinstance(null_bitmap, memoryview): + nb = null_bitmap.tobytes() + elif isinstance(null_bitmap, (bytes, bytearray)): + nb = bytes(null_bitmap) + else: + nb = bytes(null_bitmap) + + for i, col in enumerate(schema): + if nb is not None and (nb[i // 8] >> (i % 8)) & 1: + continue + code = (attributes[i // 2] & 0x0F) if i % 2 == 0 else (attributes[i // 2] >> 4) + values = col.get("values") or [] + if code < len(values): + out[col["id"]] = values[code] + + if overrides: + for k, v in overrides.items(): + if v is not None: + out[str(k)] = v if isinstance(v, str) else str(v) + return out + + +def collect_lists(decoded: dict[str, str]) -> dict[str, list[str]]: + lists: dict[str, list[str]] = {k: [] for k in LIST_FIELDS} + for field_id, value in decoded.items(): + fid = field_id.lower() + for target, prefixes in LIST_FIELDS.items(): + if any(fid == p or fid.startswith(p) for p in prefixes): + if value and value not in lists[target]: + lists[target].append(value) + return {k: v for k, v in lists.items() if v} + + +def summarize(attrs: dict[str, Any], decoded: dict[str, str]) -> str: + bits = [ + attrs.get("occupation"), + attrs.get("region"), + attrs.get("age_bracket"), + attrs.get("occupation_domain"), + ] + bits = [b for b in bits if b] + if bits: + return " · ".join(str(b) for b in bits) + # fall back to a few decoded keys + for key in ("occupation", "region", "age_bracket"): + if key in decoded: + bits.append(decoded[key]) + return " · ".join(bits) if bits else "MatrAIx persona seed" + + +def grounding_type(source: str | None) -> str: + if not source: + return "unknown" + s = source.lower() + if "synthetic" in s: + return "synthetic" + return "human_grounded" + + +def row_to_record( + *, + idx: int, + source: str | None, + source_record_id: str | None, + decoded: dict[str, str], + description: str | None, +) -> dict[str, Any]: + attrs: dict[str, Any] = {} + for src, dest in FIELD_MAP.items(): + if src in decoded and dest not in attrs: + attrs[dest] = decoded[src] + attrs.update(collect_lists(decoded)) + + rid = source_record_id or f"{source or 'row'}-{idx}" + return { + "id": str(rid), + "groundingType": grounding_type(source), + "description": description or summarize(attrs, decoded), + "attributes": attrs, + "source": source, + } + + +def find_parquet_files(dataset_dir: Path) -> list[Path]: + candidates: list[Path] = [] + for sub in ("data", "release/data", "."): + root = dataset_dir / sub if sub != "." else dataset_dir + if not root.exists(): + continue + candidates.extend(sorted(root.glob("*.parquet"))) + candidates.extend(sorted(root.glob("persona*.parquet"))) + # Prefer packed shards over sample/ flatten files + packed = [p for p in candidates if "sample" not in p.parts] + return packed or candidates + + +def description_from_row(row: dict[str, Any]) -> str | None: + descs = row.get("descriptions") + if isinstance(descs, list) and descs: + texts = [] + for d in descs[:3]: + if isinstance(d, dict): + t = d.get("text") or d.get("description") + if t: + texts.append(str(t)) + elif isinstance(d, str): + texts.append(d) + if texts: + return " ".join(texts) + if row.get("has_description") and isinstance(row.get("metadata_json"), str): + try: + meta = json.loads(row["metadata_json"]) + if isinstance(meta.get("description"), str): + return meta["description"] + except json.JSONDecodeError: + pass + return None + + +def decode_shard( + path: Path, + schema: list[dict[str, Any]], + limit: int | None, + written: int, + out_fh, +) -> int: + import pyarrow.parquet as pq + + table = pq.read_table(path) + n = table.num_rows + cols = set(table.column_names) + + for i in range(n): + if limit is not None and written >= limit: + return written + attributes = table["attributes"][i].as_py() if "attributes" in cols else None + null_bitmap = table["null_bitmap"][i].as_py() if "null_bitmap" in cols else None + overrides = None + if "attribute_overrides" in cols: + overrides = table["attribute_overrides"][i].as_py() + if isinstance(overrides, str): + try: + overrides = json.loads(overrides) + except json.JSONDecodeError: + overrides = None + + decoded = decode_attributes(attributes, null_bitmap, schema, overrides) + source = table["source"][i].as_py() if "source" in cols else None + srid = ( + table["source_record_id"][i].as_py() + if "source_record_id" in cols + else None + ) + row_dict = {name: table[name][i].as_py() for name in cols} + rec = row_to_record( + idx=written, + source=source, + source_record_id=srid, + decoded=decoded, + description=description_from_row(row_dict), + ) + out_fh.write(json.dumps(rec, ensure_ascii=False) + "\n") + written += 1 + return written + + +def jsonl_to_array(jsonl_path: Path, json_path: Path, limit: int | None) -> int: + rows = [] + with jsonl_path.open(encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + if limit is not None and len(rows) >= limit: + break + json_path.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return len(rows) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-dir", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path, help="Output .jsonl path") + parser.add_argument("--json-out", type=Path, help="Also write a JSON array for MATRAIX_CORPUS_PATH") + parser.add_argument("--limit", type=int, default=None) + args = parser.parse_args() + + try: + import pyarrow.parquet # noqa: F401 + except ImportError as exc: + raise SystemExit("pyarrow is required: pip install pyarrow") from exc + + schema = load_schema(args.dataset_dir) + files = find_parquet_files(args.dataset_dir) + if not files: + raise SystemExit(f"No parquet files under {args.dataset_dir}") + + args.out.parent.mkdir(parents=True, exist_ok=True) + written = 0 + with args.out.open("w", encoding="utf-8") as out_fh: + for path in files: + written = decode_shard(path, schema, args.limit, written, out_fh) + if args.limit is not None and written >= args.limit: + break + + print(f"wrote {written} records → {args.out}", file=sys.stderr) + if args.json_out: + n = jsonl_to_array(args.out, args.json_out, args.limit) + print(f"wrote JSON array ({n}) → {args.json_out}", file=sys.stderr) + print( + f"export MATRAIX_CORPUS_PATH={args.json_out.resolve()}", + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/skills/persona-seed/scripts/decode-matraix.sh b/skills/persona-seed/scripts/decode-matraix.sh new file mode 100755 index 0000000..5dfcb62 --- /dev/null +++ b/skills/persona-seed/scripts/decode-matraix.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Download (optional) + decode MatrAIx Persona 1M into MATRAIX_CORPUS_PATH JSON. +set -euo pipefail + +SEED_DIR="$(cd "$(dirname "$0")/.." && pwd)" +DECODE_PY="$SEED_DIR/providers/matraix-persona-1m/scripts/decode_parquet.py" +DATASET_DIR="${MATRAIX_DATASET_DIR:-./matraix-1m}" +OUT_JSONL="${MATRAIX_JSONL:-./matraix-1m.decoded.jsonl}" +OUT_JSON="${MATRAIX_CORPUS_PATH:-./matraix-1m.decoded.json}" +LIMIT="${MATRAIX_DECODE_LIMIT:-1000}" +DOWNLOAD="${MATRAIX_DOWNLOAD:-0}" + +usage() { + cat < + +Env: + MATRAIX_DATASET_DIR local HF download dir (default ./matraix-1m) + MATRAIX_CORPUS_PATH output JSON array (default ./matraix-1m.decoded.json) + MATRAIX_DECODE_LIMIT max rows (default 1000) + MATRAIX_DOWNLOAD=1 run huggingface-cli download first +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --download) DOWNLOAD=1; shift ;; + --limit) LIMIT="$2"; shift 2 ;; + --dataset-dir) DATASET_DIR="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown arg: $1" >&2; usage; exit 1 ;; + esac +done + +if [[ "$DOWNLOAD" == "1" ]]; then + if ! command -v huggingface-cli >/dev/null 2>&1; then + echo "huggingface-cli not found. Install: pip install huggingface_hub" >&2 + exit 1 + fi + huggingface-cli download MatrAIx2026/MatrAIx_Persona_1M \ + --repo-type dataset \ + --local-dir "$DATASET_DIR" +fi + +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 required" >&2 + exit 1 +fi + +python3 "$DECODE_PY" \ + --dataset-dir "$DATASET_DIR" \ + --out "$OUT_JSONL" \ + --json-out "$OUT_JSON" \ + --limit "$LIMIT" + +echo "Next:" +echo " export MATRAIX_CORPUS_PATH=$(cd "$(dirname "$OUT_JSON")" && pwd)/$(basename "$OUT_JSON")" +echo " node $SEED_DIR/scripts/prepare-corpus.js --validate \"\$MATRAIX_CORPUS_PATH\" --limit 20" +echo " node $SEED_DIR/scripts/search.js --intent '{\"domain\":[\"software\"],\"limit\":5}'" diff --git a/skills/persona-seed/tests/persona-seed.test.js b/skills/persona-seed/tests/persona-seed.test.js index 7788338..e1bbb20 100644 --- a/skills/persona-seed/tests/persona-seed.test.js +++ b/skills/persona-seed/tests/persona-seed.test.js @@ -178,3 +178,33 @@ describe('persona-seed / prepare-corpus', () => { assert.equal(result.count, 5); }); }); + +describe('persona-seed / jsonl corpus', () => { + it('search works when MATRAIX_CORPUS_PATH is jsonl', () => { + const jsonlPath = path.join(os.tmpdir(), `persona-seed-corpus-${Date.now()}.jsonl`); + const fixture = path.join( + __dirname, + '..', + 'providers', + 'matraix-persona-1m', + 'fixtures', + 'sample-corpus.json' + ); + const rows = JSON.parse(fs.readFileSync(fixture, 'utf8')); + fs.writeFileSync(jsonlPath, rows.map((r) => JSON.stringify(r)).join('\n') + '\n'); + const prev = process.env.MATRAIX_CORPUS_PATH; + process.env.MATRAIX_CORPUS_PATH = jsonlPath; + matraix.resetCorpusCache(); + try { + const caps = matraix.capabilities(); + assert.equal(caps.corpusMode, 'external'); + const hits = matraix.search({ domain: ['software'], traits: ['precise'], limit: 3 }); + assert.ok(hits.some((h) => h.id === 'fixture-001')); + } finally { + if (prev === undefined) delete process.env.MATRAIX_CORPUS_PATH; + else process.env.MATRAIX_CORPUS_PATH = prev; + matraix.resetCorpusCache(); + fs.rmSync(jsonlPath, { force: true }); + } + }); +}); From e58775baddcd50619f4e4fa30d6579e0d5443eae Mon Sep 17 00:00:00 2001 From: NeilJo-GY <43027886+NeilJo-GY@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:23:15 +0800 Subject: [PATCH 3/4] Add persona-seed provider registry driven by schema families Resolve providers via providers/registry.json (id or hfRepo) so /datasets listings stay discovery-only until explicitly registered, and future corpora share family adapters instead of hard-coded maps. Co-authored-by: Cursor --- skills/persona-seed/SKILL.md | 13 ++- skills/persona-seed/providers/README.md | 20 ++-- skills/persona-seed/providers/registry.json | 24 +++++ .../references/provider-contract.md | 24 ++++- skills/persona-seed/scripts/registry.js | 102 ++++++++++++++++++ skills/persona-seed/scripts/run-pipeline.js | 8 +- skills/persona-seed/scripts/search.js | 85 +++++++++++---- .../persona-seed/tests/persona-seed.test.js | 27 +++++ 8 files changed, 265 insertions(+), 38 deletions(-) create mode 100644 skills/persona-seed/providers/registry.json create mode 100644 skills/persona-seed/scripts/registry.js diff --git a/skills/persona-seed/SKILL.md b/skills/persona-seed/SKILL.md index bf7befe..004d911 100644 --- a/skills/persona-seed/SKILL.md +++ b/skills/persona-seed/SKILL.md @@ -39,7 +39,8 @@ user intent Contract: `references/provider-contract.md` Schemas: `schemas/intent.schema.json`, `schemas/seed-profile.schema.json` -Providers: `providers/` (add new sources there only) +Registry: `providers/registry.json` (hfRepo / provider id → module; `/datasets` publish ≠ seed-capable) +Providers: `providers//` modules, registered explicitly ## Tools (scripts) @@ -47,7 +48,10 @@ Set `SEED_DIR` to this skill directory (path containing `SKILL.md`). | Task | Command | |------|---------| +| List registry | `node ${SEED_DIR}/scripts/search.js --list-providers` | +| Seed-capable HF repos | `node ${SEED_DIR}/scripts/search.js --seed-capable` | | List provider caps | `node ${SEED_DIR}/scripts/search.js --capabilities` | +| Caps by HF repo | `node ${SEED_DIR}/scripts/search.js --repo MatrAIx2026/MatrAIx_Persona_1M --capabilities` | | Search seeds | `node ${SEED_DIR}/scripts/search.js --intent ''` | | Fetch raw | `node ${SEED_DIR}/scripts/search.js --fetch --id ` | | To SeedProfile | `node ${SEED_DIR}/scripts/search.js --to-seed --id ` | @@ -143,6 +147,7 @@ Writes `soul/seed-provenance.json` only — does **not** add unknown root keys t ## Adding another corpus later -1. Add `providers//` implementing the contract. -2. Register in `scripts/search.js` → `PROVIDERS`. -3. No new orchestration skill — reuse this one. +1. Prefer an existing **family** adapter if the schema matches. +2. Otherwise add `providers//` implementing the contract. +3. Register in `providers/registry.json` (`hfRepos`, `family`, `status`). +4. No new orchestration skill — reuse this one. `/datasets` listing alone is not enough. diff --git a/skills/persona-seed/providers/README.md b/skills/persona-seed/providers/README.md index eb2b1fd..90a17dd 100644 --- a/skills/persona-seed/providers/README.md +++ b/skills/persona-seed/providers/README.md @@ -1,14 +1,18 @@ # Seed providers -Each directory is one **Persona Seed** adapter. Orchestration lives in `../SKILL.md`; providers only implement the contract in `../references/provider-contract.md`. +Adapters that turn corpus rows into `SeedProfile`. Orchestration: `../SKILL.md`. Contract: `../references/provider-contract.md`. -| Provider id | Status | Notes | -|-------------|--------|-------| -| `matraix-persona-1m` | v1 | Attribute filter + fixture corpus; set `MATRAIX_CORPUS_PATH` for a larger decoded JSON array | +**Source of truth for what is seed-capable:** [`registry.json`](./registry.json) + +| Provider id | Family | Status | HF repos | +|-------------|--------|--------|----------| +| `matraix-persona-1m` | attribute-census | ga | `MatrAIx2026/MatrAIx_Persona_1M` | + +`openpersona.co/datasets` may list many more corpora. Only registry rows are usable with persona-seed. ## Add a provider -1. Copy `matraix-persona-1m/` as a template. -2. Implement `capabilities`, `search`, `fetch`, `toSeed` in `provider.js`. -3. Register in `../scripts/search.js` → `PROVIDERS`. -4. Keep a tiny offline fixture for tests. +1. Implement `providers//provider.js` (`capabilities`, `search`, `fetch`, `toSeed`). +2. Prefer reusing a **family** module when the schema matches an existing adapter. +3. Register in `registry.json` (`hfRepos`, `directoryUrls`, `status`). +4. Keep an offline fixture for tests. diff --git a/skills/persona-seed/providers/registry.json b/skills/persona-seed/providers/registry.json new file mode 100644 index 0000000..e8a9258 --- /dev/null +++ b/skills/persona-seed/providers/registry.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": "0.1.0", + "defaultProvider": "matraix-persona-1m", + "families": { + "attribute-census": "Structured demographic / psychographic attribute rows → SeedProfile", + "persona-card": "Name/bio/personality card rows → SeedProfile", + "dialogue-extract": "Dialogue corpora; extract persona then SeedProfile (secondary)", + "generic-hf": "Ad-hoc HF column mapping (experimental)" + }, + "providers": [ + { + "id": "matraix-persona-1m", + "family": "attribute-census", + "name": "MatrAIx Persona 1M", + "module": "./matraix-persona-1m/provider.js", + "hfRepos": ["MatrAIx2026/MatrAIx_Persona_1M"], + "directoryUrls": [ + "https://openpersona.co/datasets/matraix2026/matraix-persona-1m" + ], + "status": "ga", + "notes": "Fixture offline by default; set MATRAIX_CORPUS_PATH for decoded JSON/JSONL." + } + ] +} diff --git a/skills/persona-seed/references/provider-contract.md b/skills/persona-seed/references/provider-contract.md index 29e0d96..61da89d 100644 --- a/skills/persona-seed/references/provider-contract.md +++ b/skills/persona-seed/references/provider-contract.md @@ -1,6 +1,24 @@ # Persona Seed — Provider Contract -All corpus / population sources implement the same interface. MatrAIx Persona 1M is the first provider; new sources add an adapter under `providers//` only. +`/datasets` discovers Hugging Face corpora. **persona-seed** consumes only sources listed in `providers/registry.json`. MatrAIx is the first `attribute-census` provider — not a special-case architecture. + +## Registry + +`providers/registry.json` maps: + +- `id` → provider module +- `family` → schema family (`attribute-census` | `persona-card` | `dialogue-extract` | `generic-hf`) +- `hfRepos[]` → Hugging Face dataset ids (one provider may cover many repos with the same schema) +- `directoryUrls[]` → optional openpersona.co/datasets links +- `status` → `ga` | `experimental` | `planned` + +```bash +node scripts/search.js --list-providers +node scripts/search.js --seed-capable +node scripts/search.js --repo MatrAIx2026/MatrAIx_Persona_1M --capabilities +``` + +Publishing to `/datasets` does **not** register a seed provider. Add a registry entry + module first. ## Interface @@ -32,10 +50,12 @@ All corpus / population sources implement the same interface. MatrAIx Persona 1M 3. Sensitive or weakly evidenced attributes go into `constraints.sensitiveFlags` or are omitted — not silently asserted as facts. 4. `toSeed` must populate `gaps` for anything the shared mapper cannot invent (at least `personaName`, `slug`, `role` when absent). 5. Provenance is written by `scripts/write-provenance.js` to `soul/seed-provenance.json` after pack generation (not as a `persona.json` root key). +6. Prefer **one module per schema family**; register multiple `hfRepos` when schemas align. ## Adding a provider 1. Create `providers//provider.js` exporting `{ capabilities, search, fetch, toSeed }`. 2. Document field mapping in `providers//mapping.md`. -3. Register the id in `scripts/search.js` `PROVIDERS` map. +3. Add an entry to `providers/registry.json` (id, family, module, hfRepos, status). 4. Add a small fixture so offline tests pass without network. +5. Do **not** edit a hard-coded map in `search.js` — resolution is registry-driven. diff --git a/skills/persona-seed/scripts/registry.js b/skills/persona-seed/scripts/registry.js new file mode 100644 index 0000000..5dac851 --- /dev/null +++ b/skills/persona-seed/scripts/registry.js @@ -0,0 +1,102 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const REGISTRY_PATH = path.join(__dirname, '..', 'providers', 'registry.json'); + +let cached = null; + +function loadRegistry(registryPath = REGISTRY_PATH) { + if (cached && registryPath === REGISTRY_PATH) return cached; + const raw = JSON.parse(fs.readFileSync(registryPath, 'utf8')); + if (!Array.isArray(raw.providers)) { + throw new Error('persona-seed registry.json missing providers[]'); + } + const byId = new Map(); + const byHf = new Map(); + for (const entry of raw.providers) { + if (!entry.id || !entry.module) { + throw new Error('registry provider entries require id and module'); + } + if (byId.has(entry.id)) { + throw new Error(`duplicate provider id in registry: ${entry.id}`); + } + byId.set(entry.id, entry); + for (const repo of entry.hfRepos || []) { + const key = String(repo).toLowerCase(); + if (byHf.has(key)) { + throw new Error(`hfRepo ${repo} mapped to multiple providers`); + } + byHf.set(key, entry); + } + } + const loaded = { raw, byId, byHf, path: registryPath }; + if (registryPath === REGISTRY_PATH) cached = loaded; + return loaded; +} + +function resetRegistryCache() { + cached = null; +} + +function listProviders(opts = {}) { + const { raw } = loadRegistry(opts.registryPath); + return raw.providers.filter((p) => { + if (opts.status && p.status !== opts.status) return false; + if (opts.family && p.family !== opts.family) return false; + return true; + }); +} + +function resolveEntry(idOrRepo, opts = {}) { + const reg = loadRegistry(opts.registryPath); + if (!idOrRepo) { + const def = reg.raw.defaultProvider; + const entry = reg.byId.get(def); + if (!entry) throw new Error(`defaultProvider "${def}" not in registry`); + return entry; + } + if (reg.byId.has(idOrRepo)) return reg.byId.get(idOrRepo); + const hfKey = String(idOrRepo).toLowerCase(); + if (reg.byHf.has(hfKey)) return reg.byHf.get(hfKey); + throw new Error( + `Unknown provider or hfRepo "${idOrRepo}". Known ids: ${[...reg.byId.keys()].join(', ')}` + ); +} + +function loadProviderModule(entry, opts = {}) { + const base = path.dirname(opts.registryPath || REGISTRY_PATH); + const modPath = path.resolve(base, entry.module); + // Fresh require path resolution; cache is node module cache (fine for CLI) + // eslint-disable-next-line import/no-dynamic-require, global-require + return require(modPath); +} + +function resolveProvider(idOrRepo, opts = {}) { + const entry = resolveEntry(idOrRepo, opts); + const mod = loadProviderModule(entry, opts); + return { entry, provider: mod }; +} + +function seedCapableRepos(opts = {}) { + return listProviders(opts).flatMap((p) => + (p.hfRepos || []).map((repo) => ({ + repo, + providerId: p.id, + family: p.family, + status: p.status, + directoryUrls: p.directoryUrls || [], + })) + ); +} + +module.exports = { + REGISTRY_PATH, + loadRegistry, + resetRegistryCache, + listProviders, + resolveEntry, + resolveProvider, + seedCapableRepos, +}; diff --git a/skills/persona-seed/scripts/run-pipeline.js b/skills/persona-seed/scripts/run-pipeline.js index cf724a3..59ef295 100644 --- a/skills/persona-seed/scripts/run-pipeline.js +++ b/skills/persona-seed/scripts/run-pipeline.js @@ -9,13 +9,14 @@ * * Or pick a record: * node run-pipeline.js --id fixture-001 --name "Seeded Nova" --slug seeded-nova --out /tmp/op-seed-out + * node run-pipeline.js --repo MatrAIx2026/MatrAIx_Persona_1M --id fixture-001 --name "…" --out /tmp/out */ const fs = require('fs'); const path = require('path'); const os = require('os'); -const { resolveProvider } = require('./search'); +const { resolveProviderWithEntry } = require('./search'); const { mapSeedToPersona } = require('./map-seed-to-persona'); const { writeProvenance } = require('./write-provenance'); @@ -27,6 +28,7 @@ function parseArgs(argv) { a === '--intent' || a === '--id' || a === '--provider' || + a === '--repo' || a === '--name' || a === '--slug' || a === '--role' || @@ -44,7 +46,7 @@ function parseArgs(argv) { } async function run(args) { - const provider = resolveProvider(args.provider); + const { entry, provider } = resolveProviderWithEntry(args.provider || args.repo); let recordId = args.id; if (!recordId) { @@ -110,6 +112,8 @@ async function run(args) { return { recordId, + providerId: entry.id, + family: entry.family, seedPath, personaPath, skillDir, diff --git a/skills/persona-seed/scripts/search.js b/skills/persona-seed/scripts/search.js index f0ea0eb..166f871 100644 --- a/skills/persona-seed/scripts/search.js +++ b/skills/persona-seed/scripts/search.js @@ -1,28 +1,25 @@ #!/usr/bin/env node 'use strict'; -const matraix = require('../providers/matraix-persona-1m/provider'); - -const PROVIDERS = { - [matraix.PROVIDER_ID]: matraix, -}; - -function resolveProvider(id) { - const key = id || matraix.PROVIDER_ID; - const provider = PROVIDERS[key]; - if (!provider) { - throw new Error( - `Unknown provider "${key}". Registered: ${Object.keys(PROVIDERS).join(', ')}` - ); - } - return provider; -} +const fs = require('fs'); +const { + listProviders, + resolveProvider, + seedCapableRepos, + loadRegistry, +} = require('./registry'); function parseArgs(argv) { const out = { _: [] }; for (let i = 0; i < argv.length; i++) { const a = argv[i]; - if (a === '--intent' || a === '--id' || a === '--provider' || a === '--out') { + if ( + a === '--intent' || + a === '--id' || + a === '--provider' || + a === '--repo' || + a === '--out' + ) { out[a.slice(2)] = argv[++i]; } else if (a === '--fetch') { out.fetch = true; @@ -30,6 +27,10 @@ function parseArgs(argv) { out.toSeed = true; } else if (a === '--capabilities') { out.capabilities = true; + } else if (a === '--list-providers') { + out.listProviders = true; + } else if (a === '--seed-capable') { + out.seedCapable = true; } else { out._.push(a); } @@ -39,10 +40,43 @@ function parseArgs(argv) { function main() { const args = parseArgs(process.argv.slice(2)); - const provider = resolveProvider(args.provider); + + if (args.listProviders) { + const reg = loadRegistry(); + process.stdout.write( + `${JSON.stringify( + { + defaultProvider: reg.raw.defaultProvider, + families: reg.raw.families, + providers: listProviders(), + }, + null, + 2 + )}\n` + ); + return; + } + + if (args.seedCapable) { + process.stdout.write(`${JSON.stringify(seedCapableRepos(), null, 2)}\n`); + return; + } + + const idOrRepo = args.provider || args.repo; + const { entry, provider } = resolveProvider(idOrRepo); if (args.capabilities) { - process.stdout.write(`${JSON.stringify(provider.capabilities(), null, 2)}\n`); + const caps = { + ...provider.capabilities(), + registry: { + id: entry.id, + family: entry.family, + status: entry.status, + hfRepos: entry.hfRepos || [], + directoryUrls: entry.directoryUrls || [], + }, + }; + process.stdout.write(`${JSON.stringify(caps, null, 2)}\n`); return; } @@ -57,12 +91,14 @@ function main() { let intent = {}; if (args.intent) { - intent = JSON.parse(args.intent.startsWith('{') ? args.intent : require('fs').readFileSync(args.intent, 'utf8')); + intent = JSON.parse( + args.intent.startsWith('{') ? args.intent : fs.readFileSync(args.intent, 'utf8') + ); } else if (args._[0]) { intent = { query: args._.join(' '), limit: 5 }; } - if (args.provider) intent.provider = args.provider; + if (entry.id) intent.provider = entry.id; const results = provider.search(intent); process.stdout.write(`${JSON.stringify(results, null, 2)}\n`); } @@ -76,4 +112,9 @@ if (require.main === module) { } } -module.exports = { PROVIDERS, resolveProvider }; +module.exports = { + resolveProvider: (id) => resolveProvider(id).provider, + resolveProviderWithEntry: resolveProvider, + listProviders, + seedCapableRepos, +}; diff --git a/skills/persona-seed/tests/persona-seed.test.js b/skills/persona-seed/tests/persona-seed.test.js index e1bbb20..8033099 100644 --- a/skills/persona-seed/tests/persona-seed.test.js +++ b/skills/persona-seed/tests/persona-seed.test.js @@ -11,6 +11,12 @@ const { mapSeedToPersona, slugify } = require('../scripts/map-seed-to-persona'); const { writeProvenance } = require('../scripts/write-provenance'); const { run: runPipeline } = require('../scripts/run-pipeline'); const { validate } = require('../scripts/prepare-corpus'); +const { + resolveEntry, + resolveProvider, + seedCapableRepos, + listProviders, +} = require('../scripts/registry'); const { generate } = require('../../../lib/generator'); describe('persona-seed / matraix-persona-1m', () => { @@ -179,6 +185,27 @@ describe('persona-seed / prepare-corpus', () => { }); }); +describe('persona-seed / registry', () => { + it('resolves default and hfRepo to matraix-persona-1m', () => { + assert.equal(resolveEntry().id, 'matraix-persona-1m'); + assert.equal(resolveEntry('MatrAIx2026/MatrAIx_Persona_1M').id, 'matraix-persona-1m'); + assert.equal(resolveEntry('matraix2026/matraix_persona_1m').id, 'matraix-persona-1m'); + const { entry, provider } = resolveProvider('matraix-persona-1m'); + assert.equal(entry.family, 'attribute-census'); + assert.equal(provider.capabilities().id, 'matraix-persona-1m'); + }); + + it('lists seed-capable directory mapping', () => { + const rows = seedCapableRepos(); + assert.ok(rows.some((r) => r.repo === 'MatrAIx2026/MatrAIx_Persona_1M')); + assert.ok(listProviders({ status: 'ga' }).length >= 1); + }); + + it('rejects unknown provider ids', () => { + assert.throws(() => resolveEntry('not-a-provider'), /Unknown provider/); + }); +}); + describe('persona-seed / jsonl corpus', () => { it('search works when MATRAIX_CORPUS_PATH is jsonl', () => { const jsonlPath = path.join(os.tmpdir(), `persona-seed-corpus-${Date.now()}.jsonl`); From 21255759920d9f404e42826853594d7fcf76ecd7 Mon Sep 17 00:00:00 2001 From: NeilJo-GY <43027886+NeilJo-GY@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:37:36 +0800 Subject: [PATCH 4/4] Fix MatrAIx locale filter to match language codes Intent locales like zh/en now resolve against primary_language names (Chinese/English), matching toSeed's localeFromLanguage mapping. Co-authored-by: Cursor --- .../providers/matraix-persona-1m/provider.js | 29 +++++++++++++++---- .../persona-seed/tests/persona-seed.test.js | 6 ++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/skills/persona-seed/providers/matraix-persona-1m/provider.js b/skills/persona-seed/providers/matraix-persona-1m/provider.js index a1a4ce7..d1afa9f 100644 --- a/skills/persona-seed/providers/matraix-persona-1m/provider.js +++ b/skills/persona-seed/providers/matraix-persona-1m/provider.js @@ -138,7 +138,7 @@ function scoreRecord(record, intent) { } } for (const loc of asArray(intent.locale)) { - if (loc && text.includes(norm(loc))) { + if (localeMatches(loc, a.primary_language, text)) { score += 2; hits.push(`locale:${loc}`); } @@ -232,10 +232,27 @@ function fetch(id) { function localeFromLanguage(lang) { if (!lang) return []; const n = norm(lang); - if (n.includes('chinese') || n === 'zh' || n.includes('中文')) return ['zh']; - if (n.includes('english') || n === 'en') return ['en']; - if (n.includes('spanish') || n === 'es') return ['es']; - return [lang]; + if (n.includes('chinese') || n === 'zh' || n.includes('中文') || n.startsWith('zh-')) { + return ['zh', 'chinese', '中文']; + } + if (n.includes('english') || n === 'en' || n.startsWith('en-')) { + return ['en', 'english']; + } + if (n.includes('spanish') || n === 'es' || n.startsWith('es-')) { + return ['es', 'spanish']; + } + return [n]; +} + +/** True when intent locale (code or name) matches record primary_language. */ +function localeMatches(intentLocale, primaryLanguage, haystackText) { + if (!intentLocale) return false; + const want = norm(intentLocale); + const aliases = new Set(localeFromLanguage(primaryLanguage).map(norm)); + aliases.add(norm(primaryLanguage)); + if (aliases.has(want)) return true; + // Also allow raw substring on haystack for unmapped language names + return Boolean(haystackText && haystackText.includes(want)); } function toSeed(raw) { @@ -319,4 +336,6 @@ module.exports = { toSeed, resetCorpusCache, isFixtureCorpus, + localeFromLanguage, + localeMatches, }; diff --git a/skills/persona-seed/tests/persona-seed.test.js b/skills/persona-seed/tests/persona-seed.test.js index 8033099..e753aa5 100644 --- a/skills/persona-seed/tests/persona-seed.test.js +++ b/skills/persona-seed/tests/persona-seed.test.js @@ -50,6 +50,12 @@ describe('persona-seed / matraix-persona-1m', () => { assert.ok(!hits.some((h) => h.id === 'fixture-005')); }); + it('search locale codes match primary_language names', () => { + const hits = matraix.search({ locale: ['zh'], limit: 5 }); + assert.ok(hits.some((h) => h.id === 'fixture-001')); + assert.ok(hits.every((h) => h.id === 'fixture-001')); + }); + it('toSeed fills gaps and fixture provenance', () => { const seed = matraix.toSeed(matraix.fetch('fixture-001')); assert.equal(seed.schemaVersion, '0.1.0');