diff --git a/docs/usage.md b/docs/usage.md index 2e5265c..58ea9ac 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -9,21 +9,21 @@ All extraction goes through `SkillExtractorRefactored.extract_concepts`. This pa |---|---| | `data` | a pandas `DataFrame`, one row per document | | `id_column` | column holding each document's identifier; copied to the output. Default `"Research ID"` | -| `text_columns` | columns holding the text to analyse; joined with spaces for job descriptions | -| `input_type` | what kind of text the rows contain — selects the prompt | +| `text_columns` | columns holding the text to analyse; joined with spaces | +| `input_type` | what kind of text the rows contain; every type currently uses the same prompt | ### Input types | `input_type` | Use for | Text used | |---|---|---| | `"job_desc"` (or `"job_description"`) | job postings and role descriptions | every column in `text_columns` | -| `"syllabus"`, `"course_syllabus"`, `"course_syllabi"` | courses and programs | the `description` and `learning_outcomes` columns — list both in `text_columns` | +| `"syllabus"`, `"course_syllabus"`, `"course_syllabi"` | courses and programs | every column in `text_columns`, e.g. `["description", "learning_outcomes"]` | -!!! note "List both syllabus columns" - The syllabus prompt treats the course description and its learning outcomes separately, reading them - from the `description` and `learning_outcomes` columns. Only columns named in `text_columns` are read, - so pass `text_columns=["description", "learning_outcomes"]`. With only `description`, the learning - outcomes reach the model empty. +!!! note "One prompt for every input type" + Job postings and syllabi go through the same extraction prompt, so skills extracted from either can + be compared directly. The prompt is worded for job postings and tells the model to ignore branding, + benefits and legal boilerplate; course text passes through it unchanged. List every column you want + the model to read in `text_columns`. ## What to extract diff --git a/laiser/config.py b/laiser/config.py index f8037a0..59cdcc7 100644 --- a/laiser/config.py +++ b/laiser/config.py @@ -224,8 +224,6 @@ }} """ -## ISSUE: Prompt for course syllabi -SKILL_EXTRACTION_PROMPT_SYLLABUS = """ Work in progress""" KSA_EXTRACTION_PROMPT = """[INST]user **Objective:** Given a {input_desc}, complete the following tasks with structured outputs. diff --git a/laiser/services.py b/laiser/services.py index 48533d7..8ca5f66 100644 --- a/laiser/services.py +++ b/laiser/services.py @@ -24,7 +24,6 @@ KSA_EXTRACTION_PROMPT, KT_FROM_SKILLS_PROMPT, SCQF_LEVELS, - SKILL_EXTRACTION_PROMPT_SYLLABUS, ) from laiser.data_access import DataAccessLayer, FAISSIndexManager, KnowledgeFAISSIndexManager, TaskFAISSIndexManager from laiser.exceptions import InvalidInputError, LAiSERError @@ -52,18 +51,14 @@ def normalize_input_type(input_type: str) -> str: @staticmethod def build_skill_extraction_prompt(input_text: str, input_type: str) -> str: - """Build prompt for basic skill extraction""" - input_type = PromptBuilder.normalize_input_type(input_type) - if input_type == "job_desc": - extraction_prompt = COMBINED_EXTRACTION_PROMPT.format(description=input_text) - return extraction_prompt - elif input_type == "syllabus": - return SKILL_EXTRACTION_PROMPT_SYLLABUS.format( - description=input_text.get("description", ""), - learning_outcomes=input_text.get("learning_outcomes", ""), - ) - else: - raise InvalidInputError(f"Unsupported input type: {input_type}") + """Build the skill extraction prompt. + + Every input type uses the same prompt, so that skills extracted from + job postings and from syllabi stay comparable. ``input_type`` is still + validated, so an unsupported type fails here. + """ + PromptBuilder.normalize_input_type(input_type) + return COMBINED_EXTRACTION_PROMPT.format(description=input_text) @staticmethod def build_ksa_extraction_prompt( @@ -960,16 +955,11 @@ def _deduplicate(self, items: List[str], semantic_threshold: float = 0.92) -> Li return [exact_deduped[i] for i in kept_indices] def extract_raw_llm_skills(self, input_data, text_columns, input_type: str = "job_desc"): - normalized_input_type = self.prompt_builder.normalize_input_type(input_type) - if normalized_input_type == "syllabus": - prompt_input: Any = { - "description": str(input_data.get("description", "")).strip(), - "learning_outcomes": str(input_data.get("learning_outcomes", "")).strip(), - } - else: - prompt_input = " ".join(str(input_data.get(col, "")) for col in text_columns).strip() + # Every input type reads the same columns and uses the same prompt. Syllabi + # previously went to a placeholder prompt that contained no document text. + prompt_input = " ".join(str(input_data.get(col, "")) for col in text_columns).strip() extraction_prompt = self.prompt_builder.build_skill_extraction_prompt( - input_text=prompt_input, input_type=normalized_input_type + input_text=prompt_input, input_type=input_type ) response = self.router.generate(extraction_prompt) skills = self.llm_parser._parse_skills_from_response(response) diff --git a/paper.bib b/paper.bib index 4b7b2d8..ba4112b 100644 --- a/paper.bib +++ b/paper.bib @@ -1,14 +1,3 @@ -@article{Autor2013, - author = {Autor, David H. and Dorn, David}, - title = {The Growth of Low-Skill Service Jobs and the Polarization of the US Labor Market}, - journal = {American Economic Review}, - year = {2013}, - volume = {103}, - number = {5}, - pages = {1553--1597}, - doi = {10.1257/aer.103.5.1553} -} - @article{Boselli2018, author = {Boselli, Roberto and Cesarini, Mirko and Mercorio, Fabio and Mezzanzanica, Mario}, title = {Classifying online Job Advertisements through Machine Learning}, diff --git a/paper.md b/paper.md index 40156a7..e26e602 100644 --- a/paper.md +++ b/paper.md @@ -35,42 +35,40 @@ LAiSER is open-source software that reads such documents and links each skill th # Statement of need -The contemporary labor market is characterized by rapid technological change, evolving skill requirements, and growing disconnect between educational curricula and industry needs [@Autor2013]. Traditional skill extraction approaches rely on manual annotation, expert judgment, or keyword matching, all of which are labor-intensive, error-prone, and fail to capture semantic richness [@Boselli2018]. LAiSER provides a framework for automated skill extraction and taxonomy alignment, enabling researchers, educators, and workforce professionals to analyze skill demands at scale. Target users include labor economists studying occupational transitions, curriculum designers auditing program alignment with industry needs, and credentialing organizations producing standards-aligned digital badges. +Students and job seekers acquire skills through university courses, online courses and previous jobs, while many employers define in-house taxonomies for skills. Across syllabi and job descriptions, skills are just keywords, and without additional context what a skill means is left to subjective inference. Existing approaches that implement classical NLP methods often miss domain context [@Boselli2018], and works that use transformers or LLMs leave gaps in reproducibility and source grounding [@Li2023SkillGPT; @Herandi2024SkillLLM; @Nguyen2024Rethinking]. Without grounding, researchers and curriculum designers can be misled by irrelevant matches, which could negatively affect the decisions those matches inform. LAiSER is an open-source research tool for skill researchers that acts as a translation layer between the skill language of universities and industry, grounding its results in national taxonomies, with the choice of LLM left to the researcher. Every match carries its taxonomy source, link and similarity score, so users can check it. We share it so that researchers and educators can focus on understanding skills and iterating on their findings instead of building the pipeline themselves. Target users include labor economists studying occupational transitions, curriculum designers auditing program alignment with industry needs, and credentialing organizations producing standards-aligned digital badges. # State of the field -Several open-source tools address parts of the skill extraction problem. SkillNER [@SkillNER2022] uses rule-based matching against the EMSI (now Lightcast) skills database via spaCy, an open-source natural language processing library, offering fast lookups but limited ability to detect novel or context-dependent skills not present in its dictionary. Nesta's Skills Extractor Library [@Nesta2023] trains a spaCy named entity recognition (NER) model on labeled job advertisements and maps extracted phrases to ESCO or Lightcast taxonomies using sentence-transformer embeddings. While effective for UK job advertisements, its NER model requires domain-specific labeled training data and does not generalize well to other text types such as syllabi or credential descriptions. The esco-skill-extractor [@EscoExtractor2024] takes a simpler approach, embedding full input texts with a transformer and matching against ESCO entries by cosine similarity, but it lacks preprocessing and multi-model flexibility. +Several open-source tools address parts of the skill extraction problem. SkillNER [@SkillNER2022] uses rule-based matching against the EMSI (now Lightcast) skills database via spaCy, an open-source natural language processing library, offering fast lookups but limited ability to detect novel or context-dependent skills not present in its dictionary. Nesta's Skills Extractor Library [@Nesta2023] trains a spaCy named entity recognition (NER) model on labeled job advertisements and maps extracted phrases to ESCO or Lightcast taxonomies using sentence-transformer embeddings. The esco-skill-extractor, also published as ESCOX [@EscoExtractor2024; @Kavargyris2025ESCOX], embeds full input texts with a transformer and matches them against ESCO entries by cosine similarity. -LAiSER differs from these three tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER's extraction prompts instruct the model to disregard extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, llama.cpp, and the Gemini and OpenAI application programming interfaces, or APIs), allowing deployment on graphics processing unit (GPU) clusters, hosted APIs, or ordinary laptops. +LLMs have also been applied to skill extraction directly. SkillGPT [@Li2023SkillGPT] extracts skills from job descriptions with an open-source LLM and standardizes them to ESCO through vector similarity search. Skill-LLM [@Herandi2024SkillLLM] fine-tunes an LLM for skill extraction from job descriptions, and @Nguyen2024Rethinking evaluate in-context learning with LLMs on six skill extraction benchmarks, finding that LLMs trail supervised models overall but better handle syntactically complex skill mentions. -LLMs have also been applied to skill extraction directly. SkillGPT [@Li2023SkillGPT] extracts skills from job descriptions with an open-source LLM and standardizes them to ESCO through vector similarity search, and ESCOX [@Kavargyris2025ESCOX] is a tool that uses LLMs to extract ESCO skills and occupations from unstructured text. Skill-LLM [@Herandi2024SkillLLM] fine-tunes an LLM for skill extraction from job descriptions, and @Nguyen2024Rethinking evaluate in-context learning with LLMs on six skill extraction benchmarks, finding that LLMs trail supervised models overall but better handle syntactically complex skill mentions. +LAiSER's novelty is in the overall pipeline rather than any single step: it combines multi-LLM compatibility, multi-taxonomy compatibility, and a bring-your-own-model skeleton that accepts any Hugging Face model, a local model file, or the Gemini and OpenAI application programming interfaces (APIs). None of the tools above combines LLM extraction with mapping to more than one taxonomy; LAiSER maps to ESCO, the US Occupational Information Network (O\*NET), the UK Skills Classification and the Open Skills Network. We started out by integrating Nesta's library in our early prototypes to find out its issues and expand on it. At the time its NER model was not good with domain context: a term such as Apache Airflow could be matched with heating, ventilation and air conditioning (HVAC) airflow in the taxonomy. We moved on to letting an LLM read the whole document and return the full skill name, which LAiSER then matches with a sentence-transformer model; "Apache Airflow" now matches O\*NET's Apache Airflow entry. LAiSER grew from this into an umbrella tool that allows researchers to implement whichever setup they please. Like other prompted-LLM approaches, which @Nguyen2024Rethinking found trail supervised models, LAiSER has no published evaluation, and because models can drift, we recommend that users spot-check results on their own documents. # Software design -LAiSER's architecture is organized into three loosely coupled layers: data access, skill extraction, and taxonomy alignment. This separation was chosen to allow each component to evolve independently and to support diverse deployment scenarios without requiring changes to the overall pipeline. +When a user shares a job description, or a list of them, LAiSER initializes the model depending on the runtime environment and routes the text to a hosted API, an LLM on a graphics processing unit (GPU), or a smaller local model. The model extracts the skill keywords from the posting, and LAiSER finds the closest match for each in the taxonomies we indexed. The output lists the skills extracted, their taxonomy matches, similarity scores, and the source and link of each matched entry. An optional second model pass also extracts knowledge and tasks. -The extraction layer posed the most significant design trade-off. Training a custom NER model, as Nesta's library does, would yield fast inference but would require substantial labeled data for each new domain (syllabi, credentials, job postings). Instead, LAiSER delegates extraction to LLMs via a prompt engineering approach, trading some inference speed for broad domain generalizability without retraining. To mitigate vendor lock-in and hardware constraints, the extraction layer abstracts over multiple backends: vLLM for GPU clusters, HuggingFace Transformers and llama.cpp for models run locally, and the Gemini and OpenAI APIs for hosted inference. The backend is chosen when an extractor is created, from the requested model and the available hardware. On a GPU, if vLLM cannot load the requested model it first retries with a default checkpoint, and only if that also fails does it fall back to Transformers. +We use LLMs because many skill keywords carry a domain context that can only be captured by looking at the whole sentence. An LLM costs more than an NER model in speed, dollar spend and resources, but an NER setup needs labeled data for each new domain and would become a bottleneck as we scale the scope of the pipeline; LLMs scale to more domains quickly with zero-shot prompting and improve with few-shot examples. Because some researchers have credits for frontier AI models, some have high-performance computing (HPC) clusters, and some have neither, LAiSER supports hosted APIs, vLLM on GPUs, and Transformers or llama.cpp on ordinary machines. -For taxonomy alignment, the framework embeds both the extracted phrases and the taxonomy entries with a sentence-transformer model [@Reimers2019] and retrieves each phrase's closest entry from a precomputed index built with FAISS (Facebook AI Similarity Search) [@Johnson2019]. The index is a flat inner-product index over normalized embeddings, which makes retrieval exhaustive and exact rather than approximate. Exactness was a deliberate choice: at the scale of the bundled taxonomies, about 27,000 skill entries, an exhaustive scan is inexpensive relative to LLM inference in the preceding stage, and it guarantees that a given query returns the same entry and the same similarity score on every run, a precondition for the reproducibility properties described below. Because retrieval is encapsulated behind an index manager, an approximate index could be substituted without changes to the alignment logic should taxonomy size later require it. The system accepts pandas DataFrames as input and returns one row per aligned phrase, containing the extracted text, the matched taxonomy entry, its source taxonomy and link, and the similarity score. +Matching uses exact search, primarily for reproducibility: as research tooling, LAiSER should not give completely different results from run to run, since that may jeopardize downstream work. Each extracted phrase is embedded with a sentence-transformer model [@Reimers2019] and compared with every entry in a Facebook AI Similarity Search (FAISS) index [@Johnson2019]. At about 27,000 skill entries, an exhaustive search is cheap next to the LLM call, and the same phrase always returns the same entry and score. The closest entry is kept only if its similarity clears a threshold (0.60 for skills by default), so a phrase with no good match, such as a bare "Airflow", is dropped rather than mismatched. -Because LLMs sample their output, identical inputs can yield different generations across runs, and different backends can yield different generations from the same input. LAiSER addresses this in three ways. First, decoding is deterministic by default: every backend decodes greedily at temperature zero, and a fixed seed is supplied to every backend that accepts one, so runs stay reproducible even when a user deliberately raises the temperature. Temperature and seed are constructor arguments, so sampled variation is available when wanted but is never the silent default. Second, the alignment stage acts as a normalizing projection. Generated phrases are embedded with a fixed sentence-transformer model and matched by exact search against a precomputed index, so surface variation in the generated text collapses onto a closed, finite vocabulary of taxonomy entries, and different phrasings of the same competency can converge on the same aligned record, although because each phrase is matched independently this is likely rather than guaranteed. Third, every backend produces the same output schema, which makes results directly comparable across models and lets cross-model agreement be measured on the aligned columns. The test suite checks each backend's decoding defaults, the decoding parameters each backend actually receives, and that repeated alignment of a fixed input returns an identical result, similarity scores included. - -These measures reduce variability but do not eliminate it. Hosted models can change behind an API, and providers do not guarantee identical output even at temperature zero. Alignment can normalize how a skill is expressed, but it cannot recover a skill that a model failed to emit, so the number and identity of extracted skills can still differ across models. We therefore recommend that users pin the model identifier, temperature, seed, and similarity thresholds when reporting results, and treat cross-model agreement as an empirical quantity to be measured rather than assumed. +In its default configuration, LAiSER decodes at temperature zero and passes a fixed seed to every backend that accepts one, so that repeated runs with a local model should return the same output. These measures have limits. Hosted models may change under the same name, and providers do not guarantee identical output even at temperature zero; the OpenAI API also accepts no seed. Matching cannot recover a skill the model did not extract, and different models may extract different skills. The prompt asks for at most ten skills per document, a design choice to filter out noise, so long documents are summarized rather than covered exhaustively. The similarity thresholds were tuned against an internal hand-labeled sample that is not yet public. So that a batch keeps running, a document whose model call fails, or whose reply cannot be parsed, is skipped with a warning. All document types, including syllabi, use the same extraction prompt, because different prompts for different documents would give results that are hard to compare, and one prompt is easier to maintain. We therefore recommend that users record the model, temperature, seed and thresholds when reporting results, and spot-check a sample of the output. The source code is available at [https://github.com/LAiSER-Software/extract-module](https://github.com/LAiSER-Software/extract-module) under the BSD 3-Clause License, and can be installed with `pip install laiser`, or `pip install laiser[gpu]` for GPU inference. # Research impact statement -LAiSER has demonstrated realized impact across multiple institutions and funding initiatives. The framework's early prototypes contributed to securing a $250,000 grant from the Walmart Foundation and a subsequent $750,000 grant from the Gates Foundation, both supporting the broader Program on Skills, Credentials and Workforce Policy at The George Washington University (GW). LAiSER is integrated into the Credential Co-writer tool of the Massachusetts Institute of Technology (MIT) Digital Credentials Consortium [@DCC2025], where it provides skill extraction capabilities for generating standards-aligned Open Badges 3.0 credential templates in collaboration with Western Governors University, OneOrigin, and Open edX. Northeastern University has adopted LAiSER for internal academic projects involving skills analysis. The framework has been applied to a large-scale Texas syllabi skills extraction project using the OpenSyllabus Analytics API, analyzing thousands of course documents to map curriculum content to workforce skill demands. +The framework's early prototypes contributed to securing a $250,000 grant from the Walmart Foundation and a subsequent $750,000 grant from the Gates Foundation, both supporting the broader Program on Skills, Credentials and Workforce Policy at The George Washington University (GW). LAiSER is integrated into the Credential Co-writer tool of the Massachusetts Institute of Technology (MIT) Digital Credentials Consortium [@DCC2025], where it provides skill extraction capabilities for generating standards-aligned Open Badges 3.0 credential templates in collaboration with Western Governors University, OneOrigin, and Open edX. Northeastern University has adopted LAiSER for internal academic projects involving skills analysis. The framework has been applied to a large-scale Texas syllabi skills extraction project using the OpenSyllabus Analytics API, analyzing thousands of course documents to map curriculum content to workforce skill demands. LAiSER received the Bronze Award in the AI in Education (Higher Education Institutions) category at the QS Reimagine Education Awards 2025 [@QSReimagine2025], selected from over 1,600 global submissions. The project has been presented at Badge Summit 2025 (Colorado) and showcased at the GW Open Source Conference, GW Columbian College of Arts and Sciences (CCAS) Poster Day, and GW Innovation Fest. LAiSER is listed in the George Washington University Open Source Program Office project registry. A companion [cookbook repository](https://github.com/LAiSER-Software/laiser-cookbook) provides reproducible use-case notebooks, and a multi-session bootcamp series has been delivered to train researchers and practitioners on the framework. # AI usage disclosure -Generative AI tools were used during the development of LAiSER and the preparation of this paper. GitHub Copilot was used for code assistance during software development; all Copilot-suggested code was verified through automated test suites and manual code review. Google Gemini and Anthropic Claude were used to draft portions of the software documentation; all AI-generated documentation was reviewed and edited by graduate student volunteers affiliated with the project. No generative AI tools were used for the core algorithmic design, architectural decisions, or research analysis presented in this paper. The paper text was drafted by the authors with AI-assisted editing for clarity. +Generative AI was used in developing LAiSER and preparing this paper. GitHub Copilot (completion, coding agent and Autofix) and Claude Code (Claude Opus 5) wrote and refactored code and tests, all checked by the test suite and reviewed by the authors. Gemini 2.5 Pro and 3.1 Pro, Claude 3.5 Sonnet, and Claude Opus 4, 5 and 5.5 drafted portions of the documentation, which project members reviewed and edited. For the paper, Copilot's coding agent shortened a November 2025 draft, Claude 3.5 Sonnet and Claude Opus 4 drafted the March 2026 version, and Claude Code (Claude Opus 5) co-authored the September 2026 revisions. In this revision, the authors answered structured questions for the Statement of need, State of the field and Software design; Claude Code (Claude Opus 5.5) assembled those answers into paragraphs, added technical details from the code, checked claims against the code and sources, and drafted the Software design limitations paragraph, which the authors revised. The authors made all design decisions, reviewed and validated all AI-assisted content, and take full responsibility for the paper. # Acknowledgments -The authors acknowledge the George Washington University Institute of Public Policy and the Program on Skills, Credentials, and Workforce Policy for institutional support. This project was supported by grants from the Walmart Foundation and the Gates Foundation. The authors thank the GW Open Source Program Office, the MIT Digital Credentials Consortium, and the developers of HuggingFace Transformers, FAISS, and spaCy. +The authors acknowledge the George Washington University Institute of Public Policy and the Program on Skills, Credentials, and Workforce Policy for institutional support. This project was supported by grants from the Walmart Foundation and the Gates Foundation; the funders had no role in the design of the software or the writing of this paper. The authors thank the GW Open Source Program Office, the MIT Digital Credentials Consortium, and the developers of HuggingFace Transformers, FAISS, and spaCy. # References diff --git a/tests/test_syllabus_prompt.py b/tests/test_syllabus_prompt.py new file mode 100644 index 0000000..c2413ee --- /dev/null +++ b/tests/test_syllabus_prompt.py @@ -0,0 +1,42 @@ +"""Syllabi go through the same extraction prompt as job postings. + +Before this, input_type="syllabus" selected a placeholder prompt whose whole +text was " Work in progress", so the model never saw the course text. +""" + +from unittest.mock import Mock + +import pytest + +from laiser.exceptions import InvalidInputError +from laiser.services import PromptBuilder, ResponseParser, SkillExtractionService + +COURSE_TEXT = "Foundations of machine learning. Train models and interpret metrics." + + +@pytest.mark.parametrize("input_type", ["syllabus", "course_syllabus", "course_syllabi"]) +def test_syllabus_prompt_matches_job_prompt_and_contains_the_text(input_type): + syllabus_prompt = PromptBuilder.build_skill_extraction_prompt(COURSE_TEXT, input_type) + + assert COURSE_TEXT in syllabus_prompt + assert syllabus_prompt == PromptBuilder.build_skill_extraction_prompt(COURSE_TEXT, "job_desc") + + +def test_unsupported_input_type_still_fails(): + with pytest.raises(InvalidInputError): + PromptBuilder.build_skill_extraction_prompt(COURSE_TEXT, "resume") + + +def test_syllabus_extraction_reads_every_text_column(): + service = SkillExtractionService.__new__(SkillExtractionService) + service.prompt_builder = PromptBuilder() + service.llm_parser = ResponseParser() + service.router = Mock() + service.router.generate.return_value = '{"skills": ["machine learning"]}' + + row = {"description": "Foundations of machine learning.", "learning_outcomes": "Interpret metrics."} + skills = service.extract_raw_llm_skills(row, ["description", "learning_outcomes"], input_type="syllabus") + + prompt = service.router.generate.call_args.args[0] + assert "Foundations of machine learning. Interpret metrics." in prompt + assert skills == ["machine learning"]