From 8d77026c785d0344c02554187a20b408176c2315 Mon Sep 17 00:00:00 2001 From: Anket Date: Sat, 7 Nov 2026 17:00:36 -0500 Subject: [PATCH] fix: prepare taxonomy data separately from the PyPI package --- .github/workflows/deploy-prod.yml | 14 +++ .github/workflows/publish-taxonomy.yml | 28 ++++++ MANIFEST.in | 14 +-- README.md | 8 +- docs/examples.md | 2 + docs/index.md | 4 +- docs/installation.md | 24 +++-- docs/providers.md | 2 +- docs/reproducibility.md | 4 +- docs/taxonomies.md | 2 +- laiser/data_access.py | 42 ++++++++- laiser/services.py | 5 +- laiser/taxonomy-assets.json | 37 ++++++++ laiser/taxonomy_assets.py | 118 +++++++++++++++++++++++++ pyproject.toml | 17 ++-- scripts/build_taxonomy_bundles.py | 26 ++++++ tests/test_taxonomy_assets.py | 63 +++++++++++++ 17 files changed, 366 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/publish-taxonomy.yml create mode 100644 laiser/taxonomy-assets.json create mode 100644 laiser/taxonomy_assets.py create mode 100644 scripts/build_taxonomy_bundles.py create mode 100644 tests/test_taxonomy_assets.py diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 5f6024c..fc07ba3 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -59,6 +59,20 @@ jobs: done exit $STATUS + - name: Verify taxonomy release assets are available + run: | + python - <<'PYTHON' + import json + from pathlib import Path + from urllib.request import Request, urlopen + + manifest = json.loads(Path("laiser/taxonomy-assets.json").read_text()) + for kind in manifest["bundles"]: + url = f"{manifest['release_url']}/{manifest['version']}/{kind}.zip" + with urlopen(Request(url, method="HEAD"), timeout=30): # nosec B310 + print(f"Available: {url}") + PYTHON + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/publish-taxonomy.yml b/.github/workflows/publish-taxonomy.yml new file mode 100644 index 0000000..9acef7e --- /dev/null +++ b/.github/workflows/publish-taxonomy.yml @@ -0,0 +1,28 @@ +name: Publish taxonomy data + +on: + push: + tags: + - "taxonomy-v*" + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build verified bundles + run: python scripts/build_taxonomy_bundles.py "$GITHUB_REF_NAME" dist/taxonomy + + - name: Publish GitHub Release assets + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "$GITHUB_REF_NAME" dist/taxonomy/*.zip --verify-tag --latest=false \ + --title "LAiSER $GITHUB_REF_NAME" --notes-file docs/taxonomies.md diff --git a/MANIFEST.in b/MANIFEST.in index 1e393b6..70ad2a9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,12 +1,2 @@ -include laiser/public/faiss_skills.csv -include laiser/public/knowledge_df.json -include laiser/public/knowledge_taxonomy.csv -include laiser/public/knowledge_v05.index -include laiser/public/skills_df.json -include laiser/public/skills_v04.index -include laiser/public/task_taxonomy.csv -include laiser/public/tasks_df.json -include laiser/public/tasks_v05.index -exclude laiser/public/Task Statements.xlsx -exclude laiser/public/combined.csv -exclude laiser/public/skill_embeddings.npy +include laiser/taxonomy-assets.json +prune laiser/public diff --git a/README.md b/README.md index c488e98..6b5ddca 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ You give it a table of documents — job postings, course descriptions, syllabi. Because results point at shared taxonomy entries rather than free-text phrases, they can be counted and compared across sources — the skills employers ask for can be set against the skills a program teaches, or tracked across thousands of postings. -The language model can be a hosted API (Gemini or OpenAI) or a model running on your own machine, including on CPU with no API key. Taxonomy alignment always runs locally against indexes bundled with the package. +The language model can be a hosted API (Gemini or OpenAI) or a model running on your own machine, including on CPU with no API key. Taxonomy alignment always runs locally against indexes prepared during setup. ## Architecture @@ -41,7 +41,7 @@ LAiSER runs four stages for every document: 1. **Extraction** — The text is placed into a prompt for its input type and sent to the language model, which returns candidate concepts. 2. **Parsing and deduplication** — The response is parsed into phrases. Exact duplicates are dropped and near-duplicates are collapsed by embedding similarity. -3. **Taxonomy alignment** — Each phrase is embedded and searched against bundled FAISS indexes of taxonomy entries. Matches above a per-type similarity threshold are kept. +3. **Taxonomy alignment** — Each phrase is embedded and searched against local FAISS indexes of taxonomy entries. Matches above a per-type similarity threshold are kept. 4. **Output** — Matches are returned as a single table, optionally with graph edges linking knowledge areas to the tasks they enable. ## Requirements @@ -56,12 +56,14 @@ LAiSER runs four stages for every document: ```shell pip install laiser + laiser setup ``` - Install with GPU extras: ```shell pip install "laiser[gpu]" + laiser setup ``` - Install development dependencies from source: @@ -225,7 +227,7 @@ results = extractor.extract_concepts( ) ``` -When reporting results, also record the LAiSER version: it fixes the bundled taxonomy data and the embedding model used for alignment (`sentence-transformers/all-MiniLM-L6-v2`). Hosted providers do not guarantee identical output even at temperature `0.0`. See [Reproducibility](https://laiser-software.github.io/extract-module/reproducibility/) for details. +When reporting results, also record the LAiSER version: it pins the taxonomy data and the embedding model used for alignment (`sentence-transformers/all-MiniLM-L6-v2`). Hosted providers do not guarantee identical output even at temperature `0.0`. See [Reproducibility](https://laiser-software.github.io/extract-module/reproducibility/) for details. ## Try it in Google Colab diff --git a/docs/examples.md b/docs/examples.md index fe2fb6e..419eca0 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -6,12 +6,14 @@ This file contains copy-paste examples for the current refactored API. ```bash pip install laiser +laiser setup ``` GPU-enabled install: ```bash pip install "laiser[gpu]" +laiser setup ``` ## 2. Skills Only From a Job Description diff --git a/docs/index.md b/docs/index.md index 2b970c4..db80beb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,7 +17,7 @@ For every row in an input table, LAiSER runs four stages: knowledge and tasks behind them. 2. **Parsing and deduplication.** The model's response is parsed into a list of phrases. Exact duplicates are removed, then near-duplicates are collapsed using embedding similarity. -3. **Taxonomy alignment.** Each phrase is embedded and matched to its closest entry in bundled FAISS +3. **Taxonomy alignment.** Each phrase is embedded and matched to its closest entry in local FAISS indexes of taxonomy entries. Matches above a similarity threshold are kept, up to `top_k` per concept type. 4. **Output.** Matches are returned as one table with a row per match: the phrase as extracted, the @@ -27,7 +27,7 @@ The language model can be a hosted API or a model that runs on your own machine [Model providers](providers.md). The taxonomy indexes ship with the package, so alignment runs locally either way. -## Bundled taxonomies +## Supported taxonomies | Source | `allowed_sources` value | Covers | |---|---|---| diff --git a/docs/installation.md b/docs/installation.md index 0378cbe..208530f 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -10,15 +10,22 @@ ```bash pip install laiser +laiser setup ``` -This installs everything needed to extract with a hosted provider or a local transformers model, and to -align results against the bundled taxonomies. +Run `laiser setup` once before extracting. It downloads the package's taxonomy bundles from GitHub Releases, +checks their integrity, and prepares the alignment embedding model. No provider API key is needed for setup. + +The taxonomy bundles total about 125 MiB compressed and are cached under `~/.cache/laiser` by default. +Set `LAISER_CACHE_DIR` to use a different taxonomy cache directory. The embedding model uses the +Hugging Face cache, which can be set with `HF_HOME`. Use the same cache settings for setup and extraction. +Re-running setup reuses the prepared files. ### GPU inference with vLLM ```bash pip install "laiser[gpu]" +laiser setup ``` The `gpu` extra adds [vLLM](https://docs.vllm.ai/) for faster local inference on CUDA devices, plus @@ -49,12 +56,11 @@ pip install -e ".[dev]" The `dev` extra adds the test and lint tooling used in CI. -## What downloads on first use +The checkout includes the taxonomy files. Run `laiser setup` to prepare its alignment embedding model +ahead of time as well. -!!! note "First run is slower" - The taxonomy indexes are part of the package, but two things are fetched the first time they are - needed and cached afterwards by Hugging Face: +## Language models - - the sentence-embedding model used for alignment and deduplication - (`sentence-transformers/all-MiniLM-L6-v2`), and - - the language model itself, if you use a local model rather than a hosted API. +Setup prepares the taxonomy data and the embedding model used for alignment. If you choose a local +language model for extraction, that model has its own download. Hosted providers such as Gemini still +need network access when extracting from text. diff --git a/docs/providers.md b/docs/providers.md index a8a0753..1e4c76f 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -1,6 +1,6 @@ # Model providers -The language model does the extraction step; alignment always runs locally against the bundled indexes. +The language model does the extraction step; alignment always runs locally against the local indexes. Choose the model when you construct the extractor. | You pass | Runs on | Needs | diff --git a/docs/reproducibility.md b/docs/reproducibility.md index f69ab06..62916ef 100644 --- a/docs/reproducibility.md +++ b/docs/reproducibility.md @@ -21,7 +21,7 @@ For Transformers models, sampling is turned off explicitly rather than left to t ship a `generation_config.json` that enables sampling, which would otherwise make repeated runs differ. Alignment is deterministic as well. Each extracted phrase is embedded with a fixed sentence-transformer model -and matched by exact search against the bundled taxonomy index, so the same phrase always resolves to the same +and matched by exact search against the local taxonomy index, so the same phrase always resolves to the same taxonomy entry with the same similarity score. ## Changing the defaults @@ -67,7 +67,7 @@ results = extractor.extract_concepts( Also record: -- **the LAiSER version** — it fixes the bundled taxonomy data and the embedding model used for alignment, +- **the LAiSER version** — it pins the taxonomy data and the embedding model used for alignment, `sentence-transformers/all-MiniLM-L6-v2`; - **the model revision**, for Hugging Face models that may be updated under the same name. diff --git a/docs/taxonomies.md b/docs/taxonomies.md index 6d025cd..cbcd9ef 100644 --- a/docs/taxonomies.md +++ b/docs/taxonomies.md @@ -1,6 +1,6 @@ # Taxonomies -Extracted concepts are aligned against taxonomy indexes that ship inside the package. Each index is a +Extracted concepts are aligned against local taxonomy indexes prepared by `laiser setup`. Each index is a FAISS inner-product index over sentence embeddings of the entries' names and descriptions, built from the CSV files in `laiser/public/`. diff --git a/laiser/data_access.py b/laiser/data_access.py index de04967..18defd3 100644 --- a/laiser/data_access.py +++ b/laiser/data_access.py @@ -72,10 +72,17 @@ from laiser.config import COMBINED_SKILLS_URL, DEFAULT_EMBEDDING_MODEL, ESCO_SKILLS_URL, OSN_SKILLS_URL from laiser.exceptions import FAISSIndexError, LAiSERError +from laiser.taxonomy_assets import MANIFEST, is_source_checkout, prepared_bundle logger = logging.getLogger(__name__) +def load_embedding_model(*, local_files_only: bool) -> SentenceTransformer: + return SentenceTransformer( + DEFAULT_EMBEDDING_MODEL, revision=MANIFEST["embedding_revision"], local_files_only=local_files_only + ) + + class DataAccessLayer: """Handles data loading and external API calls""" @@ -89,7 +96,13 @@ def __init__(self): def get_embedding_model(self) -> SentenceTransformer: """Get or initialize the embedding model""" if self.embedding_model is None: - self.embedding_model = SentenceTransformer(DEFAULT_EMBEDDING_MODEL) + local_files_only = not is_source_checkout() + try: + self.embedding_model = load_embedding_model(local_files_only=local_files_only) + except OSError as exc: + if not local_files_only: + raise + raise LAiSERError("The alignment embedding model is missing. Run 'laiser setup'.") from exc return self.embedding_model # ------------------------- @@ -203,6 +216,14 @@ def load_faiss_index(self, file_path: str) -> Optional[faiss.IndexFlatIP]: except Exception as e: raise FAISSIndexError(f"Failed to load FAISS index: {e}") + def load_prepared_index(self, kind, index_name, metadata_name): + directory = prepared_bundle(kind) + index = self.load_faiss_index(str(directory / index_name)) + metadata = pd.read_json(directory / metadata_name, orient="records") + if index is None or index.ntotal != len(metadata): + raise FAISSIndexError(f"The {kind} index and metadata do not match at {directory}.") + return index, metadata + def download_faiss_index(self, url: str, local_path: str) -> bool: """Download FAISS index from URL""" try: @@ -249,6 +270,14 @@ def initialize_index(self, force_rebuild: bool = False, debug: bool = False) -> local_npy_path = script_dir / "public" / "skill_embeddings.npy" local_combined_csv_path = script_dir / "public" / "faiss_skills.csv" + if not is_source_checkout(): + if force_rebuild: + raise LAiSERError("Rebuilding taxonomy indexes requires a source checkout.") + self.index, self.metadata = self.data_access.load_prepared_index( + "skills", local_index_path.name, local_json_path.name + ) + return self.index, self.metadata + # 1) Try to load existing index + metadata unless force_rebuild requested if not force_rebuild: ## Issue: Embedding (npy) is not accessed. Cosine Calculations might be faster if npy is accessed. @@ -692,6 +721,7 @@ class _BaseTaxonomyFAISSIndexManager: META_FILENAME: str = "" CSV_FILENAME: str = "" LABEL: str = "" + BUNDLE: str = "" def __init__(self, data_access: DataAccessLayer): self.data_access = data_access @@ -706,6 +736,14 @@ def initialize_index(self, force_rebuild: bool = False) -> None: meta_path = script_dir / "public" / self.META_FILENAME csv_path = script_dir / "public" / self.CSV_FILENAME + if not is_source_checkout(): + if force_rebuild: + raise LAiSERError("Rebuilding taxonomy indexes requires a source checkout.") + self.index, self.metadata = self.data_access.load_prepared_index( + self.BUNDLE, self.INDEX_FILENAME, self.META_FILENAME + ) + return + # Try loading cached index if not force_rebuild and index_path.exists() and meta_path.exists(): try: @@ -821,6 +859,7 @@ class KnowledgeFAISSIndexManager(_BaseTaxonomyFAISSIndexManager): META_FILENAME = "knowledge_df.json" CSV_FILENAME = "knowledge_taxonomy.csv" LABEL = "Knowledge" + BUNDLE = "knowledge" class TaskFAISSIndexManager(_BaseTaxonomyFAISSIndexManager): @@ -830,3 +869,4 @@ class TaskFAISSIndexManager(_BaseTaxonomyFAISSIndexManager): META_FILENAME = "tasks_df.json" CSV_FILENAME = "task_taxonomy.csv" LABEL = "Task" + BUNDLE = "tasks" diff --git a/laiser/services.py b/laiser/services.py index 8ca5f66..0e7390b 100644 --- a/laiser/services.py +++ b/laiser/services.py @@ -28,6 +28,7 @@ from laiser.data_access import DataAccessLayer, FAISSIndexManager, KnowledgeFAISSIndexManager, TaskFAISSIndexManager from laiser.exceptions import InvalidInputError, LAiSERError from laiser.llm_models.llm_router import LLMRouter +from laiser.taxonomy_assets import is_source_checkout logger = logging.getLogger(__name__) @@ -645,7 +646,7 @@ def __init__( self.faiss_manager = FAISSIndexManager(self.data_access) self.alignment_service = AlignmentService(self.data_access, self.faiss_manager) - # Knowledge + Task indexes (v0.5) — initialized lazily; no-op if CSV not yet built + # Knowledge and task indexes (v0.5) self.knowledge_faiss = KnowledgeFAISSIndexManager(self.data_access) self.knowledge_faiss.initialize_index(force_rebuild=False) self.knowledge_alignment = AlignmentService(self.data_access, self.knowledge_faiss) @@ -669,6 +670,8 @@ def __init__( # Initialize skills FAISS index self.faiss_manager.initialize_index(force_rebuild=False) + if not is_source_checkout(): + self.data_access.get_embedding_model() # Log router initialization state for debugging try: llama_present = self.backend == "llama_cpp" and getattr(self.router, "llm", None) is not None diff --git a/laiser/taxonomy-assets.json b/laiser/taxonomy-assets.json new file mode 100644 index 0000000..bdbfa57 --- /dev/null +++ b/laiser/taxonomy-assets.json @@ -0,0 +1,37 @@ +{ + "version": "taxonomy-v1", + "release_url": "https://github.com/LAiSER-Software/extract-module/releases/download", + "embedding_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "bundles": { + "skills": { + "skills_v04.index": { + "size": 41702445, + "sha256": "7ce221e34d82d84834bdde281c856d4b8fbbea238d33a351b27d62fae090ddef" + }, + "skills_df.json": { + "size": 17447006, + "sha256": "a2e3af040711cbdb197156a3b4fb2fddbb9549436ab8eb40dc597e697cf4748b" + } + }, + "knowledge": { + "knowledge_v05.index": { + "size": 12761133, + "sha256": "0eca417c323aa8904f2958c7a9c06eca46f1cb1610a838615f0f3c30ff601782" + }, + "knowledge_df.json": { + "size": 3939145, + "sha256": "5f3fef364481b1ea19b88a0fafa4576286f5d375e8e9180c029aee898b6fb3fb" + } + }, + "tasks": { + "tasks_v05.index": { + "size": 80016429, + "sha256": "7386969b9ddb56303b3ffd8c7b98921ef35829b85ad679a47ee777bf6c06bb9a" + }, + "tasks_df.json": { + "size": 23418740, + "sha256": "9fb5afffc24182893e5c29cf6d7055f2d7311e72ec014fea4da6cf561900d889" + } + } + } +} diff --git a/laiser/taxonomy_assets.py b/laiser/taxonomy_assets.py new file mode 100644 index 0000000..d01dae2 --- /dev/null +++ b/laiser/taxonomy_assets.py @@ -0,0 +1,118 @@ +"""Install and locate versioned taxonomy assets.""" + +import argparse +import hashlib +import json +import os +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from tempfile import TemporaryDirectory +from zipfile import BadZipFile, ZipFile + +import requests + +from laiser.exceptions import LAiSERError + +MANIFEST = json.loads(Path(__file__).with_name("taxonomy-assets.json").read_text(encoding="utf-8")) +SOURCE_DATA = Path(__file__).with_name("public") +CHUNK_SIZE = 1024 * 1024 + + +def is_source_checkout(): + return (SOURCE_DATA.parent.parent / "scripts" / "build_task_index.py").is_file() + + +def cache_dir(): + base = os.getenv("LAISER_CACHE_DIR") or Path(os.getenv("XDG_CACHE_HOME") or Path.home() / ".cache") / "laiser" + return Path(base).expanduser() / MANIFEST["version"] + + +def _bundle_ready(directory, files): + try: + return all((directory / name).stat().st_size == info["size"] for name, info in files.items()) + except FileNotFoundError: + return False + + +def prepared_bundle(kind): + directory = cache_dir() / kind + if not _bundle_ready(directory, MANIFEST["bundles"][kind]): + raise LAiSERError(f"Taxonomy data is missing or incomplete at {directory}. Run 'laiser setup'.") + return directory + + +def _download_bundle(kind): + root = cache_dir() + destination = root / kind + files = MANIFEST["bundles"][kind] + if destination.exists(): + if _bundle_ready(destination, files): + return f"{kind}: already prepared" + # shortcut: if a prepared directory was modified externally, move it aside before repairing it. + raise LAiSERError(f"Taxonomy data at {destination} is incomplete. Move it aside and rerun 'laiser setup'.") + + url = f"{MANIFEST['release_url']}/{MANIFEST['version']}/{kind}.zip" + with TemporaryDirectory(dir=root) as temp: + staging = Path(temp) + archive = staging / "bundle.zip" + unpacked = staging / kind + unpacked.mkdir() + + # Allow ZIP overhead while bounding the size of an unexpected response. + remaining = sum(info["size"] for info in files.values()) + CHUNK_SIZE + with requests.get(url, stream=True, timeout=(10, 60)) as response, archive.open("wb") as output: + response.raise_for_status() + for chunk in response.iter_content(CHUNK_SIZE): + remaining -= len(chunk) + if remaining < 0: + raise LAiSERError(f"{kind}: taxonomy download exceeds the expected size") + output.write(chunk) + + with ZipFile(archive) as bundle: + if sorted(bundle.namelist()) != sorted(files): + raise LAiSERError(f"{kind}: unexpected files in taxonomy archive") + for name, info in files.items(): + if bundle.getinfo(name).file_size != info["size"]: + raise LAiSERError(f"{kind}: unexpected size for {name}") + digest = hashlib.sha256() + with bundle.open(name) as source, (unpacked / name).open("wb") as output: + for chunk in iter(lambda: source.read(CHUNK_SIZE), b""): + digest.update(chunk) + output.write(chunk) + if digest.hexdigest() != info["sha256"]: + raise LAiSERError(f"{kind}: checksum mismatch for {name}") + + try: + unpacked.rename(destination) + except OSError: + if not _bundle_ready(destination, files): + raise + + return f"{kind}: downloaded and verified" + + +def main(): + parser = argparse.ArgumentParser(prog="laiser") + parser.add_argument("command", choices=["setup"], help="Prepare taxonomy data and the embedding model") + parser.parse_args() + + from laiser.data_access import load_embedding_model + + try: + if is_source_checkout(): + print(f"Taxonomy data: using source checkout at {SOURCE_DATA}") + else: + cache_dir().mkdir(parents=True, exist_ok=True) + with ThreadPoolExecutor(max_workers=3) as executor: + for status in executor.map(_download_bundle, MANIFEST["bundles"]): + print(status) + print(f"Taxonomy data: {cache_dir()}") + + try: + load_embedding_model(local_files_only=True) + print("Embedding model: already prepared") + except OSError: + load_embedding_model(local_files_only=False) + print("Embedding model: downloaded") + except (LAiSERError, OSError, BadZipFile, requests.RequestException) as exc: + parser.exit(1, f"LAiSER setup failed: {exc}\n") diff --git a/pyproject.toml b/pyproject.toml index ae7868d..5266fd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "transformers>=4.20.0", "pandas>=1.3.0", "numpy>=1.21.0", - "sentence-transformers>=2.2.0", + "sentence-transformers>=3.0.0", "faiss-cpu>=1.7.0", "certifi>=2023.0.0", "requests>=2.25.0", @@ -39,6 +39,9 @@ dependencies = [ "google-genai>=0.8.0" ] +[project.scripts] +laiser = "laiser.taxonomy_assets:main" + [project.optional-dependencies] gpu = [ "vllm>=0.2.0", @@ -73,17 +76,7 @@ include-package-data = false version = {attr = "laiser.__version__"} [tool.setuptools.package-data] -laiser = [ - "public/faiss_skills.csv", - "public/knowledge_df.json", - "public/knowledge_taxonomy.csv", - "public/knowledge_v05.index", - "public/skills_df.json", - "public/skills_v04.index", - "public/task_taxonomy.csv", - "public/tasks_df.json", - "public/tasks_v05.index", -] +laiser = ["taxonomy-assets.json"] [tool.black] diff --git a/scripts/build_taxonomy_bundles.py b/scripts/build_taxonomy_bundles.py new file mode 100644 index 0000000..98b803a --- /dev/null +++ b/scripts/build_taxonomy_bundles.py @@ -0,0 +1,26 @@ +"""Build GitHub Release archives from the committed taxonomy files.""" + +import hashlib +import json +import sys +from pathlib import Path +from zipfile import ZIP_DEFLATED, ZipFile + +root = Path(__file__).resolve().parents[1] +manifest = json.loads((root / "laiser" / "taxonomy-assets.json").read_text(encoding="utf-8")) +if len(sys.argv) != 3 or sys.argv[1] != manifest["version"]: + raise SystemExit(f"Usage: {sys.argv[0]} {manifest['version']} OUTPUT_DIR") + +output = Path(sys.argv[2]) +output.mkdir(parents=True, exist_ok=True) +for kind, files in manifest["bundles"].items(): + with ZipFile(output / f"{kind}.zip", "w", compression=ZIP_DEFLATED) as bundle: + for name, expected in files.items(): + path = root / "laiser" / "public" / name + digest = hashlib.sha256() + with path.open("rb") as source, bundle.open(name, "w") as target: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + target.write(chunk) + if path.stat().st_size != expected["size"] or digest.hexdigest() != expected["sha256"]: + raise SystemExit(f"{name} does not match the manifest. Update the taxonomy version and checksums.") diff --git a/tests/test_taxonomy_assets.py b/tests/test_taxonomy_assets.py new file mode 100644 index 0000000..a063fcf --- /dev/null +++ b/tests/test_taxonomy_assets.py @@ -0,0 +1,63 @@ +import hashlib +import io +from zipfile import ZipFile + +import pytest +import requests + +from laiser import data_access, taxonomy_assets +from laiser.exceptions import LAiSERError + + +@pytest.mark.parametrize("valid", [True, False]) +def test_setup_checks_bundle_before_publishing(tmp_path, monkeypatch, valid): + contents = b"expected" + files = { + "skills.index": {"size": len(contents), "sha256": hashlib.sha256(contents).hexdigest()}, + "skills.json": {"size": len(contents), "sha256": hashlib.sha256(contents).hexdigest()}, + } + monkeypatch.setenv("LAISER_CACHE_DIR", str(tmp_path)) + monkeypatch.setitem(taxonomy_assets.MANIFEST, "bundles", {"skills": files}) + root = taxonomy_assets.cache_dir() + root.mkdir() + + archive = io.BytesIO() + with ZipFile(archive, "w") as bundle: + bundle.writestr("skills.index", contents) + bundle.writestr("skills.json", contents if valid else b"modified") + + def response(*args, **kwargs): + result = requests.Response() + result.status_code = 200 + result.raw = io.BytesIO(archive.getvalue()) + return result + + monkeypatch.setattr(taxonomy_assets.requests, "get", response) + with pytest.raises(LAiSERError, match="laiser setup"): + taxonomy_assets.prepared_bundle("skills") + + if valid: + taxonomy_assets._download_bundle("skills") + directory = taxonomy_assets.prepared_bundle("skills") + assert all((directory / name).read_bytes() == contents for name in files) + monkeypatch.setattr(taxonomy_assets.requests, "get", lambda *args, **kwargs: pytest.fail("Unexpected download")) + assert taxonomy_assets._download_bundle("skills") == "skills: already prepared" + else: + with pytest.raises(LAiSERError, match="checksum mismatch"): + taxonomy_assets._download_bundle("skills") + assert list(root.iterdir()) == [] + + +@pytest.mark.parametrize( + ("manager", "kind"), + [ + (data_access.FAISSIndexManager, "skills"), + (data_access.KnowledgeFAISSIndexManager, "knowledge"), + (data_access.TaskFAISSIndexManager, "tasks"), + ], +) +def test_installed_index_managers_require_setup(tmp_path, monkeypatch, manager, kind): + monkeypatch.setenv("LAISER_CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(data_access, "is_source_checkout", lambda: False) + with pytest.raises(LAiSERError, match=rf"{kind}\. Run 'laiser setup'"): + manager(data_access.DataAccessLayer()).initialize_index()