diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000..d4ce81a
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,67 @@
+name: Docs
+
+# Documentation lives in this repository, so it is reviewed and versioned with
+# the code it describes. Every PR that could change it builds the site in strict
+# mode; merges to main publish it to GitHub Pages.
+on:
+ pull_request:
+ paths:
+ - "docs/**"
+ - "mkdocs.yml"
+ - "laiser/**" # the API reference is generated from docstrings
+ - ".github/workflows/docs.yml"
+ push:
+ branches:
+ - main
+ paths:
+ - "docs/**"
+ - "mkdocs.yml"
+ - "laiser/**"
+ - ".github/workflows/docs.yml"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: docs-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ name: Build docs
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.11"
+
+ - name: Install docs toolchain
+ run: pip install -r docs/requirements.txt
+
+ - name: Build (strict)
+ # Broken links, missing anchors and unresolved API references fail here.
+ run: mkdocs build --strict
+
+ - name: Upload site
+ if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+ uses: actions/upload-pages-artifact@v3
+ with:
+ path: site
+
+ deploy:
+ name: Deploy to GitHub Pages
+ if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+ needs: build
+ runs-on: ubuntu-latest
+ permissions:
+ pages: write
+ id-token: write
+ environment:
+ name: github-pages
+ url: ${{ steps.deploy.outputs.page_url }}
+ steps:
+ - id: deploy
+ uses: actions/deploy-pages@v4
diff --git a/README.md b/README.md
index cf961c7..9289f15 100644
--- a/README.md
+++ b/README.md
@@ -5,48 +5,50 @@
+LAiSER turns free text such as job postings and course syllabi into structured skills, knowledge and tasks, each matched to an entry in ESCO, O\*NET, the UK Skills Classification or the Open Skills Network.
+
### Contents
-LAiSER is a tool that helps learners, educators and employers share trusted and mutually intelligible information about skills.
- [About](#about)
- [Architecture](#architecture)
- [Requirements](#requirements)
- [Setup and Installation](#setup-and-installation)
- [Usage](#usage)
+- [Try it in Google Colab](#try-it-in-google-colab)
- [Funding](#funding)
- [Authors](#authors)
- [Partners](#partners)
## About
-LAiSER is an innovative tool that harnesses the power of artificial intelligence to simplify the extraction and analysis of skills. It is designed for learners, educators, and employers who want to gain reliable insights into skill sets, ensuring that the information shared is both trusted and mutually intelligible across various sectors.
+LAiSER is a Python package for turning unstructured text about work and learning into structured, comparable skill data.
+
+You give it a table of documents — job postings, course descriptions, syllabi. For each document, a language model extracts the skills it describes and, optionally, the knowledge and tasks behind them. LAiSER then matches every extracted phrase to its closest entries in established taxonomies using embedding similarity. The result is a table with one row per match: the phrase as written, the taxonomy entry it corresponds to, which taxonomy that entry comes from, and a similarity score.
-By leveraging state-of-the-art AI models, LAiSER automates the process of identifying and classifying skills from diverse data sources. This not only saves time but also enhances accuracy, making it easier for users to discover emerging trends and in-demand skills.
+Because results point at shared taxonomy entries rather than free-text phrases, they can be counted and compared across sources — the skills employers ask for can be set against the skills a program teaches, or tracked across thousands of postings.
-The tool emphasizes standardization and transparency, offering a common framework that bridges the communication gap between different stakeholders. With LAiSER, educators can better align their teaching methods with industry requirements, and employers can more effectively identify the competencies required for their teams. The result is a more efficient and strategic approach to skill development, benefiting the entire ecosystem.
+The language model can be a hosted API (Gemini or OpenAI) or a model running on your own machine, including on CPU with no API key. Taxonomy alignment always runs locally against indexes bundled with the package.
## Architecture
-LAiSER uses a four-stage extraction and alignment pipeline:
+LAiSER runs four stages for every document:
-1. Extraction
- Input text is normalized by input type and passed through prompt construction and LLM inference to produce raw concept candidates.
-2. Parsing and deduplication
- Model output is parsed into structured concepts and filtered through exact and semantic deduplication.
-3. Taxonomy alignment
- Extracted concepts are matched against bundled taxonomy indexes using embedding-based similarity search and threshold filtering.
-4. Output normalization
- Alignment results are converted into a unified tabular schema, with optional edge generation for graph-style outputs.
+1. **Extraction** — The text is placed into a prompt for its input type and sent to the language model, which returns candidate concepts.
+2. **Parsing and deduplication** — The response is parsed into phrases. Exact duplicates are dropped and near-duplicates are collapsed by embedding similarity.
+3. **Taxonomy alignment** — Each phrase is embedded and searched against bundled FAISS indexes of taxonomy entries. Matches above a per-type similarity threshold are kept.
+4. **Output** — Matches are returned as a single table, optionally with graph edges linking knowledge areas to the tasks they enable.
## Requirements
-- Python version `>=3.10`.
-- The package supports the current tested matrix through Python `3.13`.
-- A GPU is recommended for heavy local model workflows, but API-backed extraction can run CPU-only.
-- Provider-specific environment variables may be required depending on backend:
- - `GEMINI_API_KEY` or `GOOGLE_API_KEY`
- - `OPENAI_API_KEY`
+
+- Python `>=3.10`; CI tests 3.10 through 3.13.
+- No GPU or API key is required. A GPU speeds up larger local models; hosted providers need their own API key.
+- Supported model providers and their settings are listed in the [documentation](https://laiser-software.github.io/extract-module/providers/).
## Setup and Installation
@@ -68,9 +70,8 @@ LAiSER uses a four-stage extraction and alignment pipeline:
pip install -e ".[dev]"
```
-**NOTE**: Python 3.10 or later is required. Python 3.12 or 3.13 is recommended for current development and CI parity.
-
You can check if your machine has a GPU available with:
+
```shell
python -c "import torch; print(torch.cuda.is_available())"
```
@@ -79,7 +80,40 @@ python -c "import torch; print(torch.cuda.is_available())"
LAiSER is used as a Python package. The recommended API is `SkillExtractorRefactored`.
-### Basic job description extraction
+### Without an API key
+
+This runs a small open model locally on CPU. The model downloads on first use.
+
+```python
+import pandas as pd
+
+from laiser.skill_extractor_refactored import SkillExtractorRefactored
+
+data = pd.DataFrame(
+ [
+ {
+ "Research ID": "job-001",
+ "description": "Build production machine learning systems in Python.",
+ }
+ ]
+)
+
+extractor = SkillExtractorRefactored(model_id="Qwen/Qwen2.5-0.5B-Instruct", use_gpu=False)
+
+results = extractor.extract_concepts(
+ data=data,
+ id_column="Research ID",
+ text_columns=["description"],
+ input_type="job_desc",
+ concepts=["skills"],
+)
+
+print(results[["Raw Concept", "Taxonomy Concept", "Taxonomy Source", "Correlation Coefficient"]])
+```
+
+Small local models are convenient for trying LAiSER out; hosted or larger models extract more reliably.
+
+### Job description extraction with Gemini
```python
import os
@@ -152,22 +186,29 @@ print(results.head())
### Common runtime options
-- `model_id`
- Provider or model selector such as `gemini` or `openai`
-- `api_key`
- API key for hosted providers
-- `use_gpu`
- Enables GPU-backed initialization where supported
-- `allowed_sources`
- Filters alignment sources. Available sources are `["esco"]`, `["onet"]`, `["osn"]`, and `["ukos"]`.
-- `top_k`
- Per-alignment-call cap for matched rows
-- `return_edges`
- Returns `{nodes, edges}` instead of only normalized rows
-- `output_csv_path`
- Writes CSV output only when explicitly provided
-
-Additional examples are available in [docs/examples.md](docs/examples.md).
+| Option | Description |
+|---|---|
+| `model_id` | `"gemini"`, `"openai"`, or a Hugging Face model id to run locally |
+| `api_key` | API key for hosted providers |
+| `use_gpu` | run local models on GPU where available |
+| `backend` | `"llama_cpp"` to run a local GGUF model |
+| `concepts` | `["skills"]` (default), or add `"knowledge"` and `"tasks"` |
+| `allowed_sources` | taxonomies to align against: `"esco"`, `"onet"`, `"osn"`, `"ukos"` |
+| `top_k` | maximum aligned matches per concept type, per document (default 25) |
+| `return_edges` | return `{nodes, edges}` instead of only the results table |
+| `output_csv_path` | also write the results to this CSV file |
+
+Every option is described in the [usage guide](https://laiser-software.github.io/extract-module/usage/), and more snippets are in [docs/examples.md](docs/examples.md).
+
+## Try it in Google Colab
+
+The [cookbook](https://github.com/LAiSER-Software/laiser-cookbook) has complete analyses that open directly in Colab:
+
+| Notebook | |
+|---|---|
+| Job skill analysis for job seekers | [](https://colab.research.google.com/github/LAiSER-Software/laiser-cookbook/blob/main/recipes/Job_Skill_Analysis/Job_Skill_Analysis_for_Job_Seekers.ipynb) |
+| University program skill analysis | [](https://colab.research.google.com/github/LAiSER-Software/laiser-cookbook/blob/main/recipes/University_Program_Skill_Analysis/University_Program_Skill_Analysis.ipynb) |
+| Pay equity analysis | [](https://colab.research.google.com/github/LAiSER-Software/laiser-cookbook/blob/main/recipes/Pay_Equity_Analysis/Pay_Equity_Analysis_Based_on_Skills.ipynb) |
## Funding
diff --git a/docs/cookbook.md b/docs/cookbook.md
new file mode 100644
index 0000000..06ed7fd
--- /dev/null
+++ b/docs/cookbook.md
@@ -0,0 +1,36 @@
+# Cookbook notebooks
+
+The [laiser-cookbook](https://github.com/LAiSER-Software/laiser-cookbook) repository holds complete
+analyses built on LAiSER. Each opens directly in Google Colab, which is the quickest way to try the package
+without a local install.
+
+!!! tip "Running in Colab"
+ The notebooks use a hosted model, so add your provider key as a Colab secret (the key icon in the
+ left sidebar) before running. To avoid needing a key, switch the extractor to a small local model as
+ shown in the [Quickstart](quickstart.md) — expect slower extraction on Colab's CPU runtime.
+
+## Job skill analysis
+
+Extracts and visualizes the skills requested across a set of job postings, aimed at job seekers deciding
+what to learn next.
+
+[](https://colab.research.google.com/github/LAiSER-Software/laiser-cookbook/blob/main/recipes/Job_Skill_Analysis/Job_Skill_Analysis_for_Job_Seekers.ipynb)
+
+[View on GitHub](https://github.com/LAiSER-Software/laiser-cookbook/tree/main/recipes/Job_Skill_Analysis)
+
+## University program skill analysis
+
+Extracts and categorizes the skills described in university program descriptions, for comparing what
+programs teach.
+
+[](https://colab.research.google.com/github/LAiSER-Software/laiser-cookbook/blob/main/recipes/University_Program_Skill_Analysis/University_Program_Skill_Analysis.ipynb)
+
+[View on GitHub](https://github.com/LAiSER-Software/laiser-cookbook/tree/main/recipes/University_Program_Skill_Analysis)
+
+## Pay equity analysis
+
+Compares compensation against the skills each role actually requires, rather than job titles alone.
+
+[](https://colab.research.google.com/github/LAiSER-Software/laiser-cookbook/blob/main/recipes/Pay_Equity_Analysis/Pay_Equity_Analysis_Based_on_Skills.ipynb)
+
+[View on GitHub](https://github.com/LAiSER-Software/laiser-cookbook/tree/main/recipes/Pay_Equity_Analysis)
diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 0000000..2b970c4
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,47 @@
+# LAiSER
+
+**LAiSER** (Leveraging Artificial Intelligence for Skill Extraction & Research) is a Python package
+that reads free text — job postings, course descriptions, syllabi — and returns the skills, knowledge
+areas and tasks it describes, each matched to an entry in a published taxonomy.
+
+The point of the matching step is comparability. Two job postings that say "Python scripting" and
+"writing Python code" should end up pointing at the same taxonomy skill, so the results can be counted,
+joined and compared across employers, programs and datasets.
+
+## How it works
+
+For every row in an input table, LAiSER runs four stages:
+
+1. **Extraction.** The document text is placed into a prompt for its input type (job description or
+ syllabus) and sent to a language model, which returns candidate skills — and, if requested, the
+ knowledge and tasks behind them.
+2. **Parsing and deduplication.** The model's response is parsed into a list of phrases. Exact
+ duplicates are removed, then near-duplicates are collapsed using embedding similarity.
+3. **Taxonomy alignment.** Each phrase is embedded and matched to its closest entry in bundled FAISS
+ indexes of taxonomy entries. Matches above a similarity threshold are kept, up to `top_k` per concept
+ type.
+4. **Output.** Matches are returned as one table with a row per match: the phrase as extracted, the
+ taxonomy entry it matched, which taxonomy that entry comes from, and the similarity score.
+
+The language model can be a hosted API or a model that runs on your own machine — see
+[Model providers](providers.md). The taxonomy indexes ship with the package, so alignment runs locally
+either way.
+
+## Bundled taxonomies
+
+| Source | `allowed_sources` value | Covers |
+|---|---|---|
+| ESCO — European Skills, Competences, Qualifications and Occupations | `esco` | skills, knowledge, tasks |
+| O\*NET — US Occupational Information Network | `onet` | skills, knowledge, tasks |
+| UK Skills Classification | `ukos` | skills, knowledge, tasks |
+| Open Skills Network | `osn` | skills |
+
+Details and entry counts are on the [Taxonomies](taxonomies.md) page.
+
+## Where to go next
+
+- **[Installation](installation.md)** — install from PyPI, with or without GPU support.
+- **[Quickstart](quickstart.md)** — a complete run, including one that needs no API key.
+- **[Usage](usage.md)** — every option of `extract_concepts` and what the output columns mean.
+- **[Cookbook notebooks](cookbook.md)** — end-to-end analyses you can open in Google Colab.
+- **[API reference](reference.md)** — generated from the source.
diff --git a/docs/installation.md b/docs/installation.md
new file mode 100644
index 0000000..0378cbe
--- /dev/null
+++ b/docs/installation.md
@@ -0,0 +1,60 @@
+# Installation
+
+## Requirements
+
+- Python **3.10 or later**. CI tests 3.10, 3.11, 3.12 and 3.13.
+- A GPU is optional. Hosted providers and small local models run on CPU; larger local models are
+ practical only on a GPU.
+
+## Install from PyPI
+
+```bash
+pip install laiser
+```
+
+This installs everything needed to extract with a hosted provider or a local transformers model, and to
+align results against the bundled taxonomies.
+
+### GPU inference with vLLM
+
+```bash
+pip install "laiser[gpu]"
+```
+
+The `gpu` extra adds [vLLM](https://docs.vllm.ai/) for faster local inference on CUDA devices, plus
+`bitsandbytes` and `accelerate`, which LAiSER uses to load a model in 8-bit when vLLM cannot. Check that
+PyTorch can see your GPU:
+
+```bash
+python -c "import torch; print(torch.cuda.is_available())"
+```
+
+### Local GGUF models with llama.cpp
+
+To run a quantized GGUF model through llama.cpp, install its Python bindings separately:
+
+```bash
+pip install llama-cpp-python
+```
+
+See [Model providers](providers.md#llamacpp-local-gguf) for how to point LAiSER at the model file.
+
+## Install from source
+
+```bash
+git clone https://github.com/LAiSER-Software/extract-module.git
+cd extract-module
+pip install -e ".[dev]"
+```
+
+The `dev` extra adds the test and lint tooling used in CI.
+
+## What downloads on first use
+
+!!! note "First run is slower"
+ The taxonomy indexes are part of the package, but two things are fetched the first time they are
+ needed and cached afterwards by Hugging Face:
+
+ - the sentence-embedding model used for alignment and deduplication
+ (`sentence-transformers/all-MiniLM-L6-v2`), and
+ - the language model itself, if you use a local model rather than a hosted API.
diff --git a/docs/providers.md b/docs/providers.md
new file mode 100644
index 0000000..afc66ae
--- /dev/null
+++ b/docs/providers.md
@@ -0,0 +1,107 @@
+# Model providers
+
+The language model does the extraction step; alignment always runs locally against the bundled indexes.
+Choose the model when you construct the extractor.
+
+| You pass | Runs on | Needs |
+|---|---|---|
+| `model_id="gemini"` | Google Gemini API | a Gemini API key, passed as `api_key` |
+| `model_id="openai"` | OpenAI API | `api_key`, or `OPENAI_API_KEY` in the environment |
+| `model_id=""`, `use_gpu=False` | your CPU, through transformers | nothing — no key, no GPU |
+| `model_id=""`, `use_gpu=True` | your GPU, through vLLM | `laiser[gpu]` and a CUDA device |
+| `backend="llama_cpp"` | your machine, through llama.cpp | `llama-cpp-python` and a GGUF file |
+
+`hf_token` is only needed for gated or private Hugging Face models.
+
+## Gemini
+
+```python
+import os
+from laiser.skill_extractor_refactored import SkillExtractorRefactored
+
+extractor = SkillExtractorRefactored(
+ model_id="gemini",
+ api_key=os.environ["GEMINI_API_KEY"],
+)
+```
+
+The key must be passed as `api_key`; it is not read from the environment automatically. Optional settings,
+read from environment variables:
+
+| Variable | Default | Effect |
+|---|---|---|
+| `LAISER_GEMINI_MODEL` | `gemini-2.5-flash` | which Gemini model to call |
+| `LAISER_GEMINI_MAX_OUTPUT_TOKENS` | `4096` | maximum length of each response |
+
+## OpenAI
+
+```python
+extractor = SkillExtractorRefactored(model_id="openai") # uses OPENAI_API_KEY
+```
+
+Requests go to the OpenAI Responses API with the `gpt-4.1-mini` model. The model is not currently
+configurable through LAiSER.
+
+## Local model on CPU
+
+Hugging Face causal language models in standard precision run on CPU with no API key. Pass the model explicitly:
+
+```python
+extractor = SkillExtractorRefactored(
+ model_id="Qwen/Qwen2.5-0.5B-Instruct",
+ use_gpu=False,
+)
+```
+
+!!! note "Always pass a `model_id` on CPU"
+ Without one, LAiSER uses `TheBloke/Mixtral-7B-Instruct-v0.1-AWQ`. That checkpoint is AWQ-quantized for
+ GPUs and does not load on CPU, so creating the extractor fails. The same is true of other checkpoints
+ quantized for GPUs.
+
+The weights download on first use and are cached by Hugging Face. Instruction-tuned models work best,
+because LAiSER formats the prompt with the model's chat template when it has one. Each response is capped at
+1,000 new tokens.
+
+!!! warning "Model size is a trade-off"
+ Small models such as the 0.5B example above run in seconds to minutes per document on a laptop CPU,
+ but extract fewer and noisier concepts than a hosted model. Larger models extract better but may be
+ too slow without a GPU. Alignment quality does not depend on the model — only which phrases reach it.
+
+## Local model on GPU
+
+With `use_gpu=True` on a machine with CUDA, LAiSER first tries to load the model with vLLM. If vLLM cannot
+load it, LAiSER loads it with transformers instead, using 8-bit quantization. Install the GPU extra, which
+includes vLLM plus the `bitsandbytes` and `accelerate` packages that 8-bit loading needs:
+
+```bash
+pip install "laiser[gpu]"
+```
+
+If transformers cannot find or read the requested checkpoint either, LAiSER tries
+`TheBloke/Mixtral-7B-Instruct-v0.1-AWQ` instead. Only a missing or unreadable checkpoint triggers that
+retry; other load errors are not retried. On CPU there is no retry.
+
+When no model can be loaded, creating the extractor raises `LAiSERError`, with the underlying exception
+attached as its `__cause__`.
+
+## llama.cpp (local GGUF)
+
+Quantized GGUF models run through llama.cpp on CPU or GPU:
+
+```bash
+pip install llama-cpp-python
+export LAISER_LLAMA_CPP_MODEL_PATH=/path/to/model.gguf
+```
+
+```python
+extractor = SkillExtractorRefactored(backend="llama_cpp")
+```
+
+| Variable | Default | Effect |
+|---|---|---|
+| `LAISER_LLAMA_CPP_MODEL_PATH` | *(required)* | path to the `.gguf` model file |
+| `LLAMA_CPP_CTX` | `4096` | context window in tokens |
+| `LLAMA_CPP_THREADS` | llama.cpp default | CPU threads to use |
+
+!!! note "Anthropic"
+ The package contains an Anthropic client, but it is not yet selectable through `model_id`.
diff --git a/docs/quickstart.md b/docs/quickstart.md
new file mode 100644
index 0000000..dd3d765
--- /dev/null
+++ b/docs/quickstart.md
@@ -0,0 +1,106 @@
+# Quickstart
+
+This walks through one complete extraction. Pick a model below — the first needs nothing but the package.
+
+## 1. Prepare your input
+
+LAiSER takes a pandas `DataFrame` with an ID column and one or more text columns:
+
+```python
+import pandas as pd
+
+data = pd.DataFrame(
+ [
+ {
+ "Research ID": "job-001",
+ "description": "Build production machine learning systems in Python. "
+ "Deploy models with Docker and monitor them on AWS.",
+ }
+ ]
+)
+```
+
+## 2. Create an extractor
+
+=== "Local model (no API key)"
+
+ Runs on CPU. The model downloads on first use.
+
+ ```python
+ from laiser.skill_extractor_refactored import SkillExtractorRefactored
+
+ extractor = SkillExtractorRefactored(
+ model_id="Qwen/Qwen2.5-0.5B-Instruct",
+ use_gpu=False,
+ )
+ ```
+
+=== "Gemini"
+
+ ```python
+ import os
+
+ from laiser.skill_extractor_refactored import SkillExtractorRefactored
+
+ extractor = SkillExtractorRefactored(
+ model_id="gemini",
+ api_key=os.environ["GEMINI_API_KEY"],
+ )
+ ```
+
+=== "OpenAI"
+
+ ```python
+ from laiser.skill_extractor_refactored import SkillExtractorRefactored
+
+ extractor = SkillExtractorRefactored(model_id="openai") # reads OPENAI_API_KEY
+ ```
+
+Other options, including GPU and llama.cpp, are on [Model providers](providers.md).
+
+## 3. Extract and align
+
+```python
+results = extractor.extract_concepts(
+ data=data,
+ id_column="Research ID",
+ text_columns=["description"],
+ input_type="job_desc",
+ concepts=["skills"],
+)
+
+print(results[["Raw Concept", "Taxonomy Concept", "Taxonomy Source", "Correlation Coefficient"]])
+```
+
+## 4. Read the results
+
+`results` is a `DataFrame` with one row per extracted phrase that matched a taxonomy entry:
+
+| Column | Meaning |
+|---|---|
+| `Research ID` | the document the match came from |
+| `Type` | skill, knowledge or task |
+| `Raw Concept` | the phrase the model extracted from the text |
+| `Taxonomy Concept` | the taxonomy entry it matched |
+| `Taxonomy Description` | that entry's description |
+| `Taxonomy Source` | which taxonomy: `esco`, `onet`, `osn` or `ukos` |
+| `Source Url` | link to the entry, where the taxonomy provides one |
+| `Correlation Coefficient` | similarity between phrase and entry; higher is closer |
+
+Each phrase is matched to its single closest taxonomy entry and kept only if that match clears the similarity
+threshold, so a document never produces more rows than the model extracted phrases.
+
+For the input above, the local 0.5B model on a laptop CPU returned:
+
+| Type | Raw Concept | Taxonomy Concept | Taxonomy Source | Correlation Coefficient |
+|---|---|---|---|---:|
+| skill | Python programming | Program in Python | `ukos` | 0.70 |
+| skill | Docker | Docker | `onet` | 0.75 |
+
+Model output varies between runs and between models. A hosted or larger model typically extracts more
+concepts from the same text — this small model missed "machine learning" and "AWS".
+
+## Next steps
+
+- Extract knowledge and tasks too, or restrict taxonomies — see [Usage](usage.md).
+- Run a full analysis in the browser with the [Cookbook notebooks](cookbook.md).
diff --git a/docs/reference.md b/docs/reference.md
new file mode 100644
index 0000000..6289777
--- /dev/null
+++ b/docs/reference.md
@@ -0,0 +1,14 @@
+# API reference
+
+`SkillExtractorRefactored` is the public entry point. Construct it once with a model choice, then call
+`extract_concepts` on as many tables as you like; the model and taxonomy indexes are loaded once and
+reused.
+
+This page is generated from the docstrings in
+[`laiser/skill_extractor_refactored.py`](https://github.com/LAiSER-Software/extract-module/blob/main/laiser/skill_extractor_refactored.py).
+
+::: laiser.skill_extractor_refactored.SkillExtractorRefactored
+ options:
+ members:
+ - extract_concepts
+ - extract_and_align
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 0000000..896a3e2
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,6 @@
+# Documentation build only. The API reference is generated by static analysis,
+# so the laiser package and its model dependencies are not needed to build docs.
+# MkDocs 2.0 removes the plugin system that mkdocstrings and Material depend on.
+mkdocs>=1.6,<2
+mkdocs-material>=9.5
+mkdocstrings[python]>=0.26
diff --git a/docs/taxonomies.md b/docs/taxonomies.md
new file mode 100644
index 0000000..5e8021e
--- /dev/null
+++ b/docs/taxonomies.md
@@ -0,0 +1,55 @@
+# Taxonomies
+
+Extracted concepts are aligned against taxonomy indexes that ship inside the package. Each index is a
+FAISS inner-product index over sentence embeddings of the entries' names and descriptions, built from the
+CSV files in `laiser/public/`.
+
+## Sources
+
+| `allowed_sources` | Taxonomy | Entries link to |
+|---|---|---|
+| `esco` | [ESCO](https://esco.ec.europa.eu/) — European Skills, Competences, Qualifications and Occupations | `data.europa.eu/esco/...` |
+| `onet` | [O\*NET](https://www.onetcenter.org/) — US Occupational Information Network (database 30.2) | `onetcenter.org` |
+| `ukos` | [UK Skills Classification](https://skillsclassification.org/) | `skillsclassification.org` |
+| `osn` | [Open Skills Network](https://www.openskillsnetwork.org/) | `osmt.wgu.edu` |
+
+## Entries per concept type
+
+| Source | Skills | Knowledge | Tasks |
+|---|---:|---:|---:|
+| `esco` | 10,715 | 3,219 | 10,715 |
+| `onet` | 8,810 | 33 | 18,796 |
+| `ukos` | 6,693 | 5,056 | 22,583 |
+| `osn` | 932 | — | — |
+
+The Open Skills Network contributes skills only, so `allowed_sources=["osn"]` returns no knowledge or task
+matches.
+
+## Choosing sources
+
+Pass `allowed_sources` to `extract_concepts` to restrict alignment. Omit it to search every source.
+
+```python
+results = extractor.extract_concepts(
+ data=data,
+ id_column="Research ID",
+ text_columns=["description"],
+ concepts=["skills", "knowledge", "tasks"],
+ allowed_sources=["esco", "ukos"],
+)
+```
+
+Values are case- and whitespace-insensitive. `uk` is accepted as an alias for `ukos`.
+
+### Older source names
+
+Before version 1.0.1, taxonomy entries were labelled by provider *and* type, such as `esco_task` or
+`onet_knowledge`. Those names are still accepted and map onto their provider:
+
+| Accepted | Treated as |
+|---|---|
+| `esco_knowledge`, `esco_task` | `esco` |
+| `onet_knowledge`, `onet_task`, `onet_skill`, `onet_tech` | `onet` |
+
+Because concept type is chosen with `concepts`, `allowed_sources=["esco_task"]` now matches every ESCO
+entry of the requested types, not only tasks.
diff --git a/docs/usage.md b/docs/usage.md
new file mode 100644
index 0000000..2e5265c
--- /dev/null
+++ b/docs/usage.md
@@ -0,0 +1,138 @@
+# Usage
+
+All extraction goes through `SkillExtractorRefactored.extract_concepts`. This page covers its options; the
+[API reference](reference.md) lists the full signature.
+
+## Input
+
+| Parameter | Description |
+|---|---|
+| `data` | a pandas `DataFrame`, one row per document |
+| `id_column` | column holding each document's identifier; copied to the output. Default `"Research ID"` |
+| `text_columns` | columns holding the text to analyse; joined with spaces for job descriptions |
+| `input_type` | what kind of text the rows contain — selects the prompt |
+
+### Input types
+
+| `input_type` | Use for | Text used |
+|---|---|---|
+| `"job_desc"` (or `"job_description"`) | job postings and role descriptions | every column in `text_columns` |
+| `"syllabus"`, `"course_syllabus"`, `"course_syllabi"` | courses and programs | the `description` and `learning_outcomes` columns — list both in `text_columns` |
+
+!!! note "List both syllabus columns"
+ The syllabus prompt treats the course description and its learning outcomes separately, reading them
+ from the `description` and `learning_outcomes` columns. Only columns named in `text_columns` are read,
+ so pass `text_columns=["description", "learning_outcomes"]`. With only `description`, the learning
+ outcomes reach the model empty.
+
+## What to extract
+
+`concepts` selects the concept types:
+
+| Value | Returns |
+|---|---|
+| `["skills"]` *(default)* | skills only — one model call per document |
+| `["skills", "knowledge", "tasks"]` or `["all"]` | skills, plus the knowledge and tasks behind each skill — a second model call per document |
+
+```python
+results = extractor.extract_concepts(
+ data=data,
+ id_column="Research ID",
+ text_columns=["description"],
+ input_type="job_desc",
+ concepts=["skills", "knowledge", "tasks"],
+)
+```
+
+## Controlling alignment
+
+| Parameter | Default | Description |
+|---|---|---|
+| `allowed_sources` | all | taxonomies to search — see [Taxonomies](taxonomies.md) |
+| `top_k` | `25` | maximum aligned matches per concept type, per document — up to 75 rows with all three types |
+| `similarity_thresholds` | skill `0.60`, knowledge `0.50`, task `0.50` | minimum similarity per type, e.g. `{"task": 0.6}` |
+| `similarity_threshold` | *(unset)* | one minimum for every type; `similarity_thresholds` still overrides the types it names |
+
+Raise a threshold for fewer, closer matches; lower it to see weaker ones.
+
+## Output
+
+By default `extract_concepts` returns a `DataFrame` with one row per aligned phrase. Each extracted phrase is
+matched to its single closest taxonomy entry and kept only if that match clears the threshold for its type.
+
+| Column | Meaning |
+|---|---|
+| `Research ID` | the document the match came from |
+| `Type` | skill, knowledge or task |
+| `Raw Concept` | the phrase the model extracted |
+| `Taxonomy Concept` | the taxonomy entry it matched |
+| `Taxonomy Description` | that entry's description |
+| `Taxonomy Source` | `esco`, `onet`, `osn` or `ukos` |
+| `Source Url` | link to the entry, where available |
+| `Correlation Coefficient` | similarity between phrase and entry; higher is closer |
+
+!!! note "One schema for every concept type"
+ Each concept type is aligned separately, into its own columns (`Raw Skill` / `Taxonomy Skill`,
+ `Raw Knowledge` / `Taxonomy Knowledge`, `Raw Task` / `Taxonomy Task`). Before results are returned,
+ those are merged into `Raw Concept` and `Taxonomy Concept`, with `Type` recording which kind each row
+ is. Filter on `Type` to get one kind back, for example `results[results["Type"] == "skill"]`.
+
+### Writing to CSV
+
+```python
+results = extractor.extract_concepts(
+ data=data,
+ id_column="Research ID",
+ text_columns=["description"],
+ output_csv_path="results.csv",
+)
+```
+
+The file is written only when `output_csv_path` is given; the `DataFrame` is returned either way.
+
+### Graph output
+
+With `return_edges=True` the result is a dict of two `DataFrame`s:
+
+```python
+graph = extractor.extract_concepts(
+ data=data,
+ id_column="Research ID",
+ text_columns=["description"],
+ concepts=["skills", "knowledge", "tasks"],
+ return_edges=True,
+)
+
+nodes = graph["nodes"] # the normal results table
+edges = graph["edges"] # Research ID, Skill, Knowledge, Task, Edge Type, confidence
+```
+
+Each edge is an `ENABLES` relationship from a knowledge area to a task, derived from knowledge and tasks the
+model attributed to the same skill. Knowledge and tasks come from a second model call, which only runs when
+`concepts` includes `"knowledge"` or `"tasks"` — with `concepts=["skills"]` the edges table is empty.
+
+## Other options
+
+| Parameter | Default | Description |
+|---|---|---|
+| `batch_size` | `32` | accepted for compatibility; currently has no effect — rows are processed one at a time |
+| `levels` | `False` | accepted for compatibility; currently has no effect |
+| `extract` | `None` | older name for `concepts`; passing both with different values raises `ValueError` |
+| `timing` | `False` | accepted for compatibility with benchmark scripts; currently has no effect |
+| `warnings` | `False` | print a warning for each row that raises an error; the row is skipped either way |
+
+## Skills-only wrapper
+
+`extract_and_align` predates `extract_concepts` and is kept for existing code. It behaves like
+`extract_concepts` with `concepts=["skills"]`:
+
+```python
+results = extractor.extract_and_align(
+ data=data,
+ id_column="Research ID",
+ text_columns=["description"],
+ input_type="job_desc",
+)
+```
+
+More copy-paste snippets are on the [Examples](examples.md) page.
diff --git a/laiser/llm_models/hugging_face_llm.py b/laiser/llm_models/hugging_face_llm.py
index b3b03f1..ac74ee0 100644
--- a/laiser/llm_models/hugging_face_llm.py
+++ b/laiser/llm_models/hugging_face_llm.py
@@ -49,7 +49,9 @@
[1.0.0] 6/30/2025 Anket Patil Modularize LLM generation logic for transformers and vLLM
"""
-from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+from laiser.config import MAX_NEW_TOKENS
+from laiser.llm_models.model_loader import load_model_from_transformer
try:
from vllm import SamplingParams
@@ -60,25 +62,37 @@
SamplingParams = None # Optional fallback
-def llm_generate(prompt: str, tokenizer, model, model_id: str, use_gpu: bool):
+def llm_generate(
+ prompt: str, tokenizer, model, model_id: str, use_gpu: bool, max_new_tokens: int = MAX_NEW_TOKENS
+) -> str:
+ """Generate a completion with a transformers model.
+
+ Returns only the newly generated text. ``model.generate`` returns the prompt
+ tokens followed by the completion; decoding all of it hands the prompt's own
+ example JSON to the response parser, which can match it instead of the answer.
+ """
if tokenizer is None or model is None:
- quantization_config = BitsAndBytesConfig(load_in_8bit=True)
+ tokenizer, model = load_model_from_transformer(model_id, use_gpu=use_gpu)
- tokenizer = AutoTokenizer.from_pretrained(model_id, revision="main") # nosec B615
- model = AutoModelForCausalLM.from_pretrained(
- model_id, revision="main", quantization_config=quantization_config, device_map="auto" # nosec B615
+ text = prompt
+ if getattr(tokenizer, "chat_template", None):
+ # Instruct models expect their chat format; raw text is often echoed or ignored.
+ text = tokenizer.apply_chat_template(
+ [{"role": "user", "content": prompt}], tokenize=False, add_generation_prompt=True
)
- inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id
outputs = model.generate(
**inputs,
- max_new_tokens=100,
- pad_token_id=tokenizer.pad_token_id,
+ max_new_tokens=max_new_tokens,
+ pad_token_id=pad_token_id,
eos_token_id=tokenizer.eos_token_id,
)
- return tokenizer.decode(outputs[0], skip_special_tokens=True)
+ completion = outputs[0][inputs["input_ids"].shape[1] :]
+ return tokenizer.decode(completion, skip_special_tokens=True).strip()
def llm_generate_vllm(prompt, llm):
diff --git a/laiser/llm_models/llm_router.py b/laiser/llm_models/llm_router.py
index 06b318c..42956af 100644
--- a/laiser/llm_models/llm_router.py
+++ b/laiser/llm_models/llm_router.py
@@ -78,10 +78,13 @@ def gemini_generate(*args, **kwargs):
print(f"Warning: openai support not available: {e}")
try:
- from laiser.llm_models.hugging_face_llm import llm_generate_vllm
+ from laiser.llm_models.hugging_face_llm import llm_generate, llm_generate_vllm
except ImportError as e:
print(f"Warning: HuggingFace LLM support not available: {e}")
+ def llm_generate(*args, **kwargs):
+ raise ImportError("HuggingFace LLM support is not available. Please install required packages.")
+
def llm_generate_vllm(*args, **kwargs):
raise ImportError("HuggingFace LLM support is not available. Please install required packages.")
@@ -114,7 +117,13 @@ def generate(self, prompt: str, **kwargs):
print("LLMRouter: routing request to llama_cpp backend")
return llama_cpp_chat(prompt, self.llm)
- print("LLMRouter: routing request to vLLM/transformer backend")
+ # A model loaded through transformers (always on CPU, and on GPU when vLLM
+ # fails) has no vLLM engine in self.llm, so it must not go to vLLM.
+ if self.llm is None and self.model is not None and self.tokenizer is not None:
+ print("LLMRouter: routing request to transformers backend")
+ return llm_generate(prompt, self.tokenizer, self.model, self.model_id, self.use_gpu)
+
+ print("LLMRouter: routing request to vLLM backend")
return llm_generate_vllm(prompt, self.llm)
# ---------------- INIT ----------------
@@ -132,7 +141,8 @@ def _initialize_components(self):
self.llm = Llama(
model_path=str(MODEL_PATH),
n_ctx=LLAMA_CPP_CTX,
- n_threads=LLAMA_CPP_THREADS or None,
+ # Environment variables are strings; llama.cpp needs an int thread count.
+ n_threads=int(LLAMA_CPP_THREADS) if LLAMA_CPP_THREADS else None,
n_gpu_layers=-1, # Use GPU if available, else CPU
# logits_all=False,
# chat_format="chatml",
@@ -140,8 +150,10 @@ def _initialize_components(self):
print("Initialized llama.cpp CPU backend.")
return
- if self.model_id == "gemini":
- print("Using Gemini API for skill extraction...")
+ if self.model_id in ("gemini", "openai"):
+ # Hosted providers need no local weights. Without this, "openai" fell
+ # through and was looked up as a Hugging Face model id.
+ print(f"Using {self.model_id} API for skill extraction...")
return
elif self.use_gpu and torch.cuda.is_available():
@@ -155,31 +167,26 @@ def _initialize_components(self):
print(f"WARNING: vLLM initialization failed: {e}")
print("Falling back to transformer model...")
- try:
- self._initialize_transformer()
- if self.model is not None:
- print("Transformer model fallback successful!")
- return
- except Exception as e:
- print(f"WARNING: Transformer model fallback also failed: {e}")
+ self._initialize_transformer()
+ print("Transformer model fallback successful!")
+ return
else:
print("Using CPU/transformer model...")
- try:
- self._initialize_transformer()
- if self.model is not None:
- print("Transformer model initialization successful!")
- return
- except Exception as e:
- print(f"WARNING: Transformer model initialization failed: {e}")
-
- print("WARNING: No model successfully initialized.")
+ self._initialize_transformer()
+ print("Transformer model initialization successful!")
+ return
+ # A local model that fails to load must fail construction. Swallowing the error
+ # left a router with no model, whose every generate() call then failed and was
+ # silently skipped per row by extract_and_align_core.
+ except LAiSERError:
+ raise
except Exception as e:
- raise LAiSERError(f"Critical failure during component initialization: {e}")
+ raise LAiSERError(f"Could not initialize model '{self.model_id}': {e}") from e
def _initialize_vllm(self):
self.llm = load_model_from_vllm(self.model_id, self.hf_token)
def _initialize_transformer(self):
- self.tokenizer, self.model = load_model_from_transformer(self.model_id, self.hf_token)
+ self.tokenizer, self.model = load_model_from_transformer(self.model_id, self.hf_token, use_gpu=self.use_gpu)
diff --git a/laiser/llm_models/model_loader.py b/laiser/llm_models/model_loader.py
index 6e1c9af..fe1245a 100644
--- a/laiser/llm_models/model_loader.py
+++ b/laiser/llm_models/model_loader.py
@@ -49,6 +49,7 @@
[1.0.0] 6/30/2025 Anket Patil Support transformer and vLLM model initialization
"""
+import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from transformers.utils import EntryNotFoundError, RepositoryNotFoundError
@@ -64,34 +65,34 @@
DEFAULT_TRANSFORMER_MODEL_ID = "TheBloke/Mixtral-7B-Instruct-v0.1-AWQ"
-def load_model_from_transformer(model_id: str = None, token: str = ""):
+def load_model_from_transformer(model_id: str = None, token: str = "", use_gpu: bool = True):
+ """Load a causal LM and its tokenizer with transformers.
+
+ 8-bit quantization is applied only when a CUDA device is actually available:
+ bitsandbytes cannot quantize on CPU and raises before any weights load, which
+ made every CPU run fail.
+ """
model_id = model_id or DEFAULT_TRANSFORMER_MODEL_ID
- quantization_config = BitsAndBytesConfig(load_in_8bit=True)
+ on_gpu = bool(use_gpu) and torch.cuda.is_available()
+
+ def _load(name):
+ load_kwargs = {"use_auth_token": token}
+ if on_gpu:
+ load_kwargs.update(quantization_config=BitsAndBytesConfig(load_in_8bit=True), device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(name, use_auth_token=token, revision="main") # nosec B615
+ model = AutoModelForCausalLM.from_pretrained(name, revision="main", **load_kwargs) # nosec B615
+ return tokenizer, model
try:
- tokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=token, revision="main") # nosec B615
- model = AutoModelForCausalLM.from_pretrained(
- model_id,
- use_auth_token=token,
- revision="main", # nosec B615
- quantization_config=quantization_config,
- device_map="auto",
- )
+ return _load(model_id)
except (RepositoryNotFoundError, EntryNotFoundError, OSError) as e:
+ if not on_gpu:
+ # The fallback is a 7B AWQ checkpoint that cannot run on CPU either,
+ # so surface the original error instead of downloading it.
+ raise
print(f"[WARN] Failed to load model '{model_id}': {e}")
print(f"[INFO] Falling back to default model: {DEFAULT_TRANSFORMER_MODEL_ID}")
- tokenizer = AutoTokenizer.from_pretrained(
- DEFAULT_TRANSFORMER_MODEL_ID, use_auth_token=token, revision="main"
- ) # nosec B615
- model = AutoModelForCausalLM.from_pretrained(
- DEFAULT_TRANSFORMER_MODEL_ID,
- use_auth_token=token,
- revision="main", # nosec B615
- quantization_config=quantization_config,
- device_map="auto",
- )
-
- return tokenizer, model
+ return _load(DEFAULT_TRANSFORMER_MODEL_ID)
DEFAULT_VLLM_MODEL_ID = "TheBloke/Mixtral-7B-Instruct-v0.1-AWQ"
diff --git a/laiser/services.py b/laiser/services.py
index f1d5a94..f7ea7d8 100644
--- a/laiser/services.py
+++ b/laiser/services.py
@@ -668,8 +668,8 @@ def __init__(
self.faiss_manager.initialize_index(force_rebuild=False)
# Log router initialization state for debugging
try:
- llama_present = getattr(self.router, "llama_llm", None) is not None
- print(f"SkillExtractionService: router.llama_llm present: {llama_present}")
+ llama_present = self.backend == "llama_cpp" and getattr(self.router, "llm", None) is not None
+ print(f"SkillExtractionService: llama.cpp model loaded: {llama_present}")
except Exception:
pass
@@ -707,18 +707,19 @@ def extract_and_align_core(
input_type : str
Type of input data
top_k : int, optional
- Maximum number of aligned items to return per document (default: 25)
+ Maximum number of aligned matches per concept type, per document
+ (default: DEFAULT_TOP_K, 25). Each type is aligned in its own call.
similarity_threshold : float, optional
- Global minimum similarity score applied to all types unless overridden
- by similarity_thresholds. Defaults to 0.20 for backward compatibility.
+ One minimum similarity score applied to every type. When omitted, the
+ per-type defaults below apply.
similarity_thresholds : dict, optional
- Per-type similarity thresholds. Keys: "skill", "knowledge", "task".
- Overrides similarity_threshold for each specified type.
- Defaults: {"skill": 0.20, "knowledge": 0.45, "task": 0.55}
+ Per-type minimums, keys "skill", "knowledge", "task". Overrides
+ similarity_threshold for the types it names.
+ Defaults: DEFAULT_SIMILARITY_THRESHOLDS, {"skill": 0.60, "knowledge": 0.50, "task": 0.50}
levels : bool
- Whether to extract skill levels
+ Accepted for compatibility; currently has no effect.
batch_size : int
- Batch size for processing
+ Accepted for compatibility; currently has no effect.
warnings : bool
Whether to show warnings
extract : list, optional
@@ -740,7 +741,7 @@ def extract_and_align_core(
pd.DataFrame (when return_edges=False, default)
DataFrame with normalized mixed-concept rows:
Research ID, Type, Raw Concept, Taxonomy Concept,
- Taxonomy Description, Taxonomy Source, Correlation Coefficient.
+ Taxonomy Description, Taxonomy Source, Source Url, Correlation Coefficient.
dict (when return_edges=True)
{"nodes": pd.DataFrame, "edges": pd.DataFrame}
edges columns: Research ID, Skill, Knowledge, Task, Edge Type, confidence
diff --git a/laiser/skill_extractor_refactored.py b/laiser/skill_extractor_refactored.py
index 044e0a3..b6d517b 100644
--- a/laiser/skill_extractor_refactored.py
+++ b/laiser/skill_extractor_refactored.py
@@ -178,17 +178,20 @@ def extract_concepts(
input_type : str
Type of input data
top_k : int, optional
- Maximum number of aligned items to return per document (default: 25)
+ Maximum number of aligned matches per concept type, per document
+ (default: 25). With skills, knowledge and tasks all requested, one
+ document can return up to three times this many rows.
similarity_threshold : float, optional
- Global minimum similarity score applied to all types unless overridden
- by similarity_thresholds. Defaults to 0.20 for backward compatibility.
+ One minimum similarity score applied to every type. When omitted, the
+ per-type defaults below apply.
similarity_thresholds : dict, optional
- Per-type thresholds. Keys: "skill", "knowledge", "task".
- Defaults: {"skill": 0.20, "knowledge": 0.45, "task": 0.55}
+ Per-type minimums, keys "skill", "knowledge", "task". Overrides
+ similarity_threshold for the types it names.
+ Defaults: {"skill": 0.60, "knowledge": 0.50, "task": 0.50}
levels : bool
- Whether to extract skill levels
+ Accepted for compatibility; currently has no effect.
batch_size : int
- Batch size for processing
+ Accepted for compatibility; currently has no effect.
warnings : bool
Whether to show warnings
concepts : list, optional
@@ -201,7 +204,7 @@ def extract_concepts(
"edges" contains ENABLES edges (Knowledge → Task per skill).
If False (default), return a plain pd.DataFrame.
timing : bool, optional
- If True, print per-stage timing for extraction and alignment steps.
+ Accepted for compatibility with benchmark callers; currently has no effect.
output_csv_path : str, optional
If provided, write normalized results to this CSV path.
diff --git a/mkdocs.yml b/mkdocs.yml
new file mode 100644
index 0000000..a29232a
--- /dev/null
+++ b/mkdocs.yml
@@ -0,0 +1,99 @@
+site_name: LAiSER
+site_description: >-
+ Extract skills, knowledge and tasks from free text and align them to ESCO, O*NET,
+ the Open Skills Network and the UK Skills Classification.
+site_url: https://laiser-software.github.io/extract-module/
+repo_url: https://github.com/LAiSER-Software/extract-module
+repo_name: LAiSER-Software/extract-module
+edit_uri: edit/main/docs/
+copyright: Copyright © 2025 LAiSER · BSD-3-Clause
+
+theme:
+ name: material
+ features:
+ - navigation.sections
+ - navigation.top
+ - navigation.footer
+ - content.code.copy
+ - content.tabs.link
+ - content.action.edit
+ - search.suggest
+ - search.highlight
+ palette:
+ - media: "(prefers-color-scheme: light)"
+ scheme: default
+ primary: indigo
+ accent: indigo
+ toggle:
+ icon: material/weather-night
+ name: Switch to dark mode
+ - media: "(prefers-color-scheme: dark)"
+ scheme: slate
+ primary: indigo
+ accent: indigo
+ toggle:
+ icon: material/weather-sunny
+ name: Switch to light mode
+
+# Internal engineering notes live in docs/ for contributors but are not part of
+# the published user documentation.
+exclude_docs: |
+ REFACTORING_GUIDE.md
+ adding_batching_vllm.md
+ v0.5_architecture.md
+ requirements.txt
+
+nav:
+ - Home: index.md
+ - Getting started:
+ - Installation: installation.md
+ - Quickstart: quickstart.md
+ - Guides:
+ - Usage: usage.md
+ - Model providers: providers.md
+ - Taxonomies: taxonomies.md
+ - Examples: examples.md
+ - Cookbook notebooks: cookbook.md
+ - Reference:
+ - API: reference.md
+
+markdown_extensions:
+ - admonition
+ - attr_list
+ - tables
+ - toc:
+ permalink: true
+ - pymdownx.details
+ - pymdownx.highlight:
+ anchor_linenums: true
+ - pymdownx.inlinehilite
+ - pymdownx.superfences
+ - pymdownx.tabbed:
+ alternate_style: true
+
+plugins:
+ - search
+ - mkdocstrings:
+ handlers:
+ python:
+ paths: [.]
+ options:
+ docstring_style: numpy
+ show_source: false
+ show_root_heading: true
+ show_root_full_path: false
+ merge_init_into_class: true
+ members_order: source
+ separate_signature: true
+ show_signature_annotations: true
+
+# With `mkdocs build --strict` these warnings fail the build, so a broken link or
+# a missing anchor is caught in the pull request instead of on the live site.
+validation:
+ nav:
+ omitted_files: warn
+ not_found: warn
+ links:
+ not_found: warn
+ anchors: warn
+ unrecognized_links: warn
diff --git a/paper.md b/paper.md
index 4b2eccb..a4f4a8b 100644
--- a/paper.md
+++ b/paper.md
@@ -29,7 +29,7 @@ bibliography: paper.bib
# Summary
-LAiSER is an open-source Python framework for extracting, standardizing, and aligning skill information from unstructured text with established skill taxonomies. The system addresses the lack of standardized, machine-readable skill information needed to facilitate communication between learners, educators, and employers. LAiSER employs a two-stage pipeline combining large language models (LLMs) with semantic vector search using FAISS [@Johnson2019] for high-precision skill extraction and taxonomy alignment to standards such as ESCO [@ESCO2020]. The framework supports multiple computational backends, including GPU-accelerated inference, cloud APIs, and CPU-only environments, making it accessible across diverse research and operational contexts. LAiSER also supports extraction within the Knowledge, Skills, and Abilities (KSA) framework with automatic proficiency level classification using the Scottish Credit and Qualifications Framework (SCQF) [@SCQF2019]. The source code is available at [https://github.com/LAiSER-Software/extract-module](https://github.com/LAiSER-Software/extract-module) under the MIT License, installable via `pip install laiser[gpu]` or `pip install laiser[cpu]`.
+LAiSER is an open-source Python framework for extracting, standardizing, and aligning skill information from unstructured text with established skill taxonomies. The system addresses the lack of standardized, machine-readable skill information needed to facilitate communication between learners, educators, and employers. LAiSER employs a two-stage pipeline combining large language models (LLMs) with semantic vector search using FAISS [@Johnson2019] for high-precision skill extraction and taxonomy alignment to standards such as ESCO [@ESCO2020]. The framework supports multiple computational backends, including GPU-accelerated inference, cloud APIs, and CPU-only environments, making it accessible across diverse research and operational contexts. LAiSER also supports extraction within the Knowledge, Skills, and Abilities (KSA) framework with automatic proficiency level classification using the Scottish Credit and Qualifications Framework (SCQF) [@SCQF2019]. The source code is available at [https://github.com/LAiSER-Software/extract-module](https://github.com/LAiSER-Software/extract-module) under the BSD 3-Clause License, installable via `pip install laiser`, or `pip install laiser[gpu]` for GPU inference.
# Statement of need
diff --git a/pyproject.toml b/pyproject.toml
index 6f48f1c..ae7868d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,6 +42,10 @@ dependencies = [
[project.optional-dependencies]
gpu = [
"vllm>=0.2.0",
+ # When vLLM cannot load a model, LAiSER loads it with transformers in 8-bit,
+ # which needs both of these (minimums are the ones transformers checks for).
+ "bitsandbytes>=0.43.1",
+ "accelerate>=0.26.0",
]
dev = [
"pytest>=6.0",
diff --git a/tests/test_llm_router.py b/tests/test_llm_router.py
new file mode 100644
index 0000000..eb4502f
--- /dev/null
+++ b/tests/test_llm_router.py
@@ -0,0 +1,179 @@
+"""Offline tests for local-model routing. No model weights are downloaded."""
+
+import pytest
+import torch
+
+from laiser.exceptions import LAiSERError
+from laiser.llm_models import hugging_face_llm, llm_router, model_loader
+from laiser.llm_models.llm_router import LLMRouter
+
+
+class _Encoded(dict):
+ def to(self, device):
+ return self
+
+
+class FakeTokenizer:
+ pad_token_id = 0
+ eos_token_id = 9
+
+ def __init__(self, chat_template=None):
+ self.chat_template = chat_template
+ self.seen_text = None
+
+ def apply_chat_template(self, messages, tokenize, add_generation_prompt):
+ return "CHAT:" + messages[0]["content"]
+
+ def __call__(self, text, return_tensors):
+ self.seen_text = text
+ return _Encoded(input_ids=torch.tensor([[1, 2, 3]]))
+
+ def decode(self, ids, skip_special_tokens):
+ return " ".join(str(i) for i in ids.tolist())
+
+
+class FakeModel:
+ device = "cpu"
+
+ def __init__(self):
+ self.generate_kwargs = None
+
+ def generate(self, **kwargs):
+ self.generate_kwargs = kwargs
+ # The prompt tokens (1, 2, 3) come back first, then the completion.
+ return torch.tensor([[1, 2, 3, 7, 8]])
+
+
+def _bare_router(**attrs):
+ """An LLMRouter with no model loading, so routing can be tested in isolation."""
+ router = LLMRouter.__new__(LLMRouter)
+ router.model_id = "some/model"
+ router.use_gpu = False
+ router.hf_token = router.api_key = router.backend = None
+ router.llm = router.model = router.tokenizer = router.nlp = None
+ for key, value in attrs.items():
+ setattr(router, key, value)
+ return router
+
+
+def test_transformers_model_is_routed_to_transformers_generate(monkeypatch):
+ calls = []
+ monkeypatch.setattr(llm_router, "llm_generate", lambda *args, **kwargs: calls.append(args) or "ok")
+ monkeypatch.setattr(llm_router, "llm_generate_vllm", lambda *args, **kwargs: pytest.fail("routed to vLLM"))
+
+ router = _bare_router(tokenizer=FakeTokenizer(), model=FakeModel())
+
+ assert router.generate("prompt") == "ok"
+ assert calls and calls[0][0] == "prompt"
+
+
+def test_vllm_engine_still_routes_to_vllm(monkeypatch):
+ monkeypatch.setattr(llm_router, "llm_generate", lambda *args, **kwargs: pytest.fail("routed to transformers"))
+ monkeypatch.setattr(llm_router, "llm_generate_vllm", lambda prompt, llm: f"vllm:{llm}")
+
+ assert _bare_router(llm="engine").generate("prompt") == "vllm:engine"
+
+
+def test_llm_generate_returns_only_the_completion():
+ model = FakeModel()
+
+ result = hugging_face_llm.llm_generate("p", FakeTokenizer(), model, "some/model", use_gpu=False)
+
+ assert result == "7 8"
+ assert model.generate_kwargs["max_new_tokens"] == hugging_face_llm.MAX_NEW_TOKENS
+
+
+def test_llm_generate_applies_chat_template_when_present():
+ tokenizer = FakeTokenizer(chat_template="{{ template }}")
+
+ hugging_face_llm.llm_generate("describe", tokenizer, FakeModel(), "some/model", use_gpu=False)
+
+ assert tokenizer.seen_text == "CHAT:describe"
+
+
+@pytest.fixture
+def recorded_loads(monkeypatch):
+ loads = []
+
+ def fake_model_from_pretrained(name, **kwargs):
+ loads.append((name, kwargs))
+ return FakeModel()
+
+ monkeypatch.setattr(model_loader.AutoTokenizer, "from_pretrained", lambda name, **kwargs: FakeTokenizer())
+ monkeypatch.setattr(model_loader.AutoModelForCausalLM, "from_pretrained", fake_model_from_pretrained)
+ monkeypatch.setattr(model_loader, "BitsAndBytesConfig", lambda **kwargs: "8bit")
+ return loads
+
+
+def test_cpu_load_skips_quantization(monkeypatch, recorded_loads):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+
+ model_loader.load_model_from_transformer("some/model", use_gpu=True)
+
+ assert "quantization_config" not in recorded_loads[0][1]
+
+
+def test_gpu_load_keeps_quantization(monkeypatch, recorded_loads):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+
+ model_loader.load_model_from_transformer("some/model", use_gpu=True)
+
+ assert recorded_loads[0][1]["quantization_config"] == "8bit"
+
+
+def test_cpu_load_failure_raises_instead_of_falling_back(monkeypatch):
+ attempted = []
+
+ def failing_from_pretrained(name, **kwargs):
+ attempted.append(name)
+ raise OSError("not found")
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(model_loader.AutoTokenizer, "from_pretrained", failing_from_pretrained)
+
+ with pytest.raises(OSError):
+ model_loader.load_model_from_transformer("missing/model", use_gpu=False)
+ assert attempted == ["missing/model"]
+
+
+# The tests below go through LLMRouter's constructor, which is what
+# SkillExtractorRefactored builds, rather than the loader alone.
+
+
+def _failing_load(*args, **kwargs):
+ raise OSError("model could not be loaded")
+
+
+def test_cpu_router_raises_when_the_model_cannot_load(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(llm_router, "load_model_from_transformer", _failing_load)
+
+ with pytest.raises(LAiSERError, match="model could not be loaded"):
+ LLMRouter("missing/model", use_gpu=False)
+
+
+def test_gpu_router_raises_when_vllm_and_transformers_both_fail(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(llm_router, "load_model_from_vllm", _failing_load)
+ monkeypatch.setattr(llm_router, "load_model_from_transformer", _failing_load)
+
+ with pytest.raises(LAiSERError, match="model could not be loaded"):
+ LLMRouter("missing/model", use_gpu=True)
+
+
+def test_gpu_router_falls_back_to_transformers_when_vllm_fails(monkeypatch):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(llm_router, "load_model_from_vllm", _failing_load)
+ monkeypatch.setattr(llm_router, "load_model_from_transformer", lambda *a, **k: (FakeTokenizer(), FakeModel()))
+
+ router = LLMRouter("some/model", use_gpu=True)
+
+ assert router.llm is None and router.model is not None
+
+
+def test_hosted_provider_loads_no_local_weights(monkeypatch):
+ monkeypatch.setattr(llm_router, "load_model_from_transformer", lambda *a, **k: pytest.fail("loaded local weights"))
+
+ router = LLMRouter("openai", use_gpu=False)
+
+ assert router.model is None and router.llm is None