From d1a6595fc031727bfc7d54ea40b58371d0be122a Mon Sep 17 00:00:00 2001 From: phanindra-max Date: Thu, 17 Sep 2026 02:54:54 -0400 Subject: [PATCH] docs(paper): address mechanical points from JOSS paper review (#447) Define alignment, remove claims contradicted by prior LLM-based tools, drop the unevaluated "accurate", cite SkillGPT, ESCOX, Skill-LLM and Nguyen et al. (2024), link the cookbook repository, remove the unused SCQF bib entry, and replace the broken Open Skills Network link. Co-Authored-By: Claude Opus 5 --- docs/taxonomies.md | 2 +- paper.bib | 46 ++++++++++++++++++++++++++++++++++++++-------- paper.md | 10 ++++++---- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/docs/taxonomies.md b/docs/taxonomies.md index 5e8021e..b8adb30 100644 --- a/docs/taxonomies.md +++ b/docs/taxonomies.md @@ -11,7 +11,7 @@ CSV files in `laiser/public/`. | `esco` | [ESCO](https://esco.ec.europa.eu/) — European Skills, Competences, Qualifications and Occupations | `data.europa.eu/esco/...` | | `onet` | [O\*NET](https://www.onetcenter.org/) — US Occupational Information Network (database 30.2) | `onetcenter.org` | | `ukos` | [UK Skills Classification](https://skillsclassification.org/) | `skillsclassification.org` | -| `osn` | [Open Skills Network](https://www.openskillsnetwork.org/) | `osmt.wgu.edu` | +| `osn` | [Open Skills Network](https://osmt.wgu.edu/), via WGU's Open Skills Management Tool | `osmt.wgu.edu` | ## Entries per concept type diff --git a/paper.bib b/paper.bib index 284c6c5..4b7b2d8 100644 --- a/paper.bib +++ b/paper.bib @@ -47,14 +47,6 @@ @inproceedings{Reimers2019 doi = {10.18653/v1/D19-1410} } -@misc{SCQF2019, - author = {{Scottish Credit and Qualifications Framework Partnership}}, - title = {SCQF Handbook: User Guide}, - year = {2019}, - url = {https://scqf.org.uk}, - note = {Accessed: 2025-11-01} -} - @misc{SkillNER2022, author = {Aito, Anas}, title = {SkillNER: A rule-based NLP module to extract skills from text}, @@ -94,3 +86,41 @@ @misc{QSReimagine2025 url = {https://www.reimagine-education.com/}, note = {Accessed: 2026-03-06} } + +@misc{Li2023SkillGPT, + author = {Li, Nan and Kang, Bo and De Bie, Tijl}, + title = {SkillGPT: a RESTful API service for skill extraction and standardization using a Large Language Model}, + year = {2023}, + eprint = {2304.11060}, + archivePrefix = {arXiv}, + doi = {10.48550/arXiv.2304.11060} +} + +@article{Kavargyris2025ESCOX, + author = {Kavargyris, Dimitrios Christos and Georgiou, Konstantinos and Papaioannou, Eleanna and Petrakis, Konstantinos and Mittas, Nikolaos and Angelis, Lefteris}, + title = {ESCOX: A tool for skill and occupation extraction using LLMs from unstructured text}, + journal = {Software Impacts}, + year = {2025}, + volume = {25}, + pages = {100772}, + doi = {10.1016/j.simpa.2025.100772} +} + +@misc{Herandi2024SkillLLM, + author = {Herandi, Amirhossein and Li, Yitao and Liu, Zhanlin and Hu, Ximin and Cai, Xiao}, + title = {Skill-LLM: Repurposing General-Purpose LLMs for Skill Extraction}, + year = {2024}, + eprint = {2410.12052}, + archivePrefix = {arXiv}, + doi = {10.48550/arXiv.2410.12052} +} + +@inproceedings{Nguyen2024Rethinking, + author = {Nguyen, Khanh and Zhang, Mike and Montariol, Syrielle and Bosselut, Antoine}, + title = {Rethinking Skill Extraction in the Job Market Domain using Large Language Models}, + booktitle = {Proceedings of the First Workshop on Natural Language Processing for Human Resources (NLP4HR 2024)}, + year = {2024}, + pages = {27--42}, + publisher = {Association for Computational Linguistics}, + doi = {10.18653/v1/2024.nlp4hr-1.3} +} diff --git a/paper.md b/paper.md index ef039e7..40156a7 100644 --- a/paper.md +++ b/paper.md @@ -31,17 +31,19 @@ bibliography: paper.bib Job postings, course syllabi, and credential descriptions all describe skills, but each in its own words, so the same ability can be phrased many different ways. That makes it hard to count which skills employers ask for, or to check whether a course teaches what jobs require. Skill taxonomies address this by giving each skill one agreed name and identifier; the European Commission's European Skills, Competences, Qualifications and Occupations classification (ESCO) [@ESCO2020] is a widely used example. -LAiSER is open-source software that reads such documents and links each skill they mention to its entry in a taxonomy. It uses a large language model (LLM), a kind of artificial intelligence (AI) system that reads and writes text, to find the skills a document describes, then matches each one to the closest taxonomy entry, so that differently worded mentions of the same skill are counted together. It is designed for researchers studying labor markets, curriculum designers comparing programs with employer demand, and organizations that issue digital credentials. +LAiSER is open-source software that reads such documents and links each skill they mention to its entry in a taxonomy. It uses a large language model (LLM), a kind of artificial intelligence (AI) system that reads and writes text, to find the skills a document describes, then links each one to the closest taxonomy entry, a step we call alignment, so that differently worded mentions of the same skill are counted together. It is designed for researchers studying labor markets, curriculum designers comparing programs with employer demand, and organizations that issue digital credentials. # Statement of need -The contemporary labor market is characterized by rapid technological change, evolving skill requirements, and growing disconnect between educational curricula and industry needs [@Autor2013]. Traditional skill extraction approaches rely on manual annotation, expert judgment, or keyword matching, all of which are labor-intensive, error-prone, and fail to capture semantic richness [@Boselli2018]. The challenge is compounded by the absence of integrated tooling that combines skill identification with alignment to recognized taxonomies, forcing researchers to assemble fragile multi-tool pipelines. LAiSER fills this gap by providing a unified, flexible, and accurate framework for automated skill extraction and taxonomy alignment, enabling researchers, educators, and workforce professionals to analyze skill demands at scale. Target users include labor economists studying occupational transitions, curriculum designers auditing program alignment with industry needs, and credentialing organizations producing standards-aligned digital badges. +The contemporary labor market is characterized by rapid technological change, evolving skill requirements, and growing disconnect between educational curricula and industry needs [@Autor2013]. Traditional skill extraction approaches rely on manual annotation, expert judgment, or keyword matching, all of which are labor-intensive, error-prone, and fail to capture semantic richness [@Boselli2018]. LAiSER provides a framework for automated skill extraction and taxonomy alignment, enabling researchers, educators, and workforce professionals to analyze skill demands at scale. Target users include labor economists studying occupational transitions, curriculum designers auditing program alignment with industry needs, and credentialing organizations producing standards-aligned digital badges. # State of the field Several open-source tools address parts of the skill extraction problem. SkillNER [@SkillNER2022] uses rule-based matching against the EMSI (now Lightcast) skills database via spaCy, an open-source natural language processing library, offering fast lookups but limited ability to detect novel or context-dependent skills not present in its dictionary. Nesta's Skills Extractor Library [@Nesta2023] trains a spaCy named entity recognition (NER) model on labeled job advertisements and maps extracted phrases to ESCO or Lightcast taxonomies using sentence-transformer embeddings. While effective for UK job advertisements, its NER model requires domain-specific labeled training data and does not generalize well to other text types such as syllabi or credential descriptions. The esco-skill-extractor [@EscoExtractor2024] takes a simpler approach, embedding full input texts with a transformer and matching against ESCO entries by cosine similarity, but it lacks preprocessing and multi-model flexibility. -LAiSER differs from these tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER's extraction prompts instruct the model to disregard extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language, an important step that existing tools omit. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, llama.cpp, and the Gemini and OpenAI application programming interfaces, or APIs), allowing deployment on graphics processing unit (GPU) clusters, hosted APIs, or ordinary laptops. These design choices reflect a deliberate decision to build a new framework rather than extend existing tools, whose architectures are not designed to accommodate LLM-based extraction or alignment across multiple taxonomies. +LAiSER differs from these three tools in several key respects. First, it uses LLMs for skill extraction rather than rule-based or NER-based approaches, enabling it to identify skills expressed in varied natural language without requiring labeled training data. Second, LAiSER's extraction prompts instruct the model to disregard extraneous content (company branding, legal boilerplate, benefits sections) while preserving task-relevant skill language. Third, the framework supports multiple LLM backends (vLLM, HuggingFace Transformers, llama.cpp, and the Gemini and OpenAI application programming interfaces, or APIs), allowing deployment on graphics processing unit (GPU) clusters, hosted APIs, or ordinary laptops. + +LLMs have also been applied to skill extraction directly. SkillGPT [@Li2023SkillGPT] extracts skills from job descriptions with an open-source LLM and standardizes them to ESCO through vector similarity search, and ESCOX [@Kavargyris2025ESCOX] is a tool that uses LLMs to extract ESCO skills and occupations from unstructured text. Skill-LLM [@Herandi2024SkillLLM] fine-tunes an LLM for skill extraction from job descriptions, and @Nguyen2024Rethinking evaluate in-context learning with LLMs on six skill extraction benchmarks, finding that LLMs trail supervised models overall but better handle syntactically complex skill mentions. # Software design @@ -61,7 +63,7 @@ The source code is available at [https://github.com/LAiSER-Software/extract-modu LAiSER has demonstrated realized impact across multiple institutions and funding initiatives. The framework's early prototypes contributed to securing a $250,000 grant from the Walmart Foundation and a subsequent $750,000 grant from the Gates Foundation, both supporting the broader Program on Skills, Credentials and Workforce Policy at The George Washington University (GW). LAiSER is integrated into the Credential Co-writer tool of the Massachusetts Institute of Technology (MIT) Digital Credentials Consortium [@DCC2025], where it provides skill extraction capabilities for generating standards-aligned Open Badges 3.0 credential templates in collaboration with Western Governors University, OneOrigin, and Open edX. Northeastern University has adopted LAiSER for internal academic projects involving skills analysis. The framework has been applied to a large-scale Texas syllabi skills extraction project using the OpenSyllabus Analytics API, analyzing thousands of course documents to map curriculum content to workforce skill demands. -LAiSER received the Bronze Award in the AI in Education (Higher Education Institutions) category at the QS Reimagine Education Awards 2025 [@QSReimagine2025], selected from over 1,600 global submissions. The project has been presented at Badge Summit 2025 (Colorado) and showcased at the GW Open Source Conference, GW Columbian College of Arts and Sciences (CCAS) Poster Day, and GW Innovation Fest. LAiSER is listed in the George Washington University Open Source Program Office project registry. A companion cookbook repository provides reproducible use-case notebooks, and a multi-session bootcamp series has been delivered to train researchers and practitioners on the framework. +LAiSER received the Bronze Award in the AI in Education (Higher Education Institutions) category at the QS Reimagine Education Awards 2025 [@QSReimagine2025], selected from over 1,600 global submissions. The project has been presented at Badge Summit 2025 (Colorado) and showcased at the GW Open Source Conference, GW Columbian College of Arts and Sciences (CCAS) Poster Day, and GW Innovation Fest. LAiSER is listed in the George Washington University Open Source Program Office project registry. A companion [cookbook repository](https://github.com/LAiSER-Software/laiser-cookbook) provides reproducible use-case notebooks, and a multi-session bootcamp series has been delivered to train researchers and practitioners on the framework. # AI usage disclosure