From ca799844722c99961b7887a7d1fd6fad19dcadf1 Mon Sep 17 00:00:00 2001 From: fpotier Date: Thu, 21 May 2026 13:10:11 +0000 Subject: [PATCH 1/2] feat(publish): allow async custom Ajv keywords --- docs/developer-guide/ajv-extensions.md | 2 +- src/published-data/validator.service.spec.ts | 86 ++++++++++++- src/published-data/validator.service.ts | 121 +++++++++++++------ 3 files changed, 167 insertions(+), 42 deletions(-) diff --git a/docs/developer-guide/ajv-extensions.md b/docs/developer-guide/ajv-extensions.md index 10ab2c6da..dc728d06b 100644 --- a/docs/developer-guide/ajv-extensions.md +++ b/docs/developer-guide/ajv-extensions.md @@ -39,7 +39,7 @@ export const dynamicDefaults = new Map([ ### Asynchronous functions -If you need to execute asynchronous code, you should declare your function `async` (only supported for dynamicDefaults). +If you need to execute asynchronous code, you should declare your function `async`. It should return a synchronous function as ajv will not resolve any `Promise` returned by custom code. Asynchronous functions have access to additional context via the `ctx` argument: diff --git a/src/published-data/validator.service.spec.ts b/src/published-data/validator.service.spec.ts index 7da657dbe..6c8115826 100644 --- a/src/published-data/validator.service.spec.ts +++ b/src/published-data/validator.service.spec.ts @@ -6,6 +6,7 @@ import { ProposalsService } from "src/proposals/proposals.service"; import { ReadOnlyDatasetsService, ValidatorService } from "./validator.service"; import { ErrorObject } from "ajv"; +/* eslint-disable @typescript-eslint/no-explicit-any */ describe("ValidatorService", () => { let service: ValidatorService; @@ -125,7 +126,6 @@ describe("ValidatorService", () => { service = await createService({ metadataSchema: schema }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any (service as any).dynamicDefaults.set( "userDefinedFunction", () => () => 5, @@ -152,7 +152,6 @@ describe("ValidatorService", () => { service = await createService({ metadataSchema: schema }); mockDataService.count.mockImplementation(() => 6); - // eslint-disable-next-line @typescript-eslint/no-explicit-any (service as any).dynamicDefaults.set( "userDefinedAsyncFunction", async function (ctx: { datasetsService: ReadOnlyDatasetsService }) { @@ -187,4 +186,87 @@ describe("ValidatorService", () => { ); }); }); + + describe("Custom Keywords", () => { + it("should handle user-defined synchronous keyword", async () => { + const schema = { + type: "object", + properties: { + evenNumber: { + type: "number", + isEven: true, + }, + }, + }; + + service = await createService({ metadataSchema: schema }); + + (service as any).keywords = [ + { + keyword: "isEven", + validate: (schemaVal: boolean, data: number) => { + if (!schemaVal) return true; + return data % 2 === 0; + }, + }, + ]; + + const validData = { metadata: { evenNumber: 4 } }; + let errors = await service.validate(validData); + expect(errors).toBeNull(); + + const invalidData = { metadata: { evenNumber: 7 } }; + errors = await service.validate(invalidData); + expect(errors).toBeDefined(); + expect(errors![0].keyword).toBe("isEven"); + }); + + it("should handle user-defined asynchronous keyword", async () => { + const schema = { + type: "object", + properties: { + proposalId: { + type: "string", + proposalExists: true, + }, + }, + }; + + service = await createService({ metadataSchema: schema }); + + mockDataService.findOne.mockImplementation(async (id) => { + return id === "prop-123"; + }); + + const checkProposalExistence = async function (ctx: any) { + const proposalExists = await ctx.proposalService.findOne( + ctx.publishedData.metadata.proposalId, + ); + + return function () { + return proposalExists; + }; + }; + + (service as any).keywords = [ + { + keyword: "proposalExists", + validate: checkProposalExistence, + }, + ]; + + const validData = { metadata: { proposalId: "prop-123" } }; + let errors = await service.validate(validData); + expect(errors).toBeNull(); + expect(mockDataService.findOne).toHaveBeenCalledWith("prop-123"); + + const invalidData = { metadata: { proposalId: "prop-999" } }; + errors = await service.validate(invalidData); + expect(mockDataService.findOne).toHaveBeenCalledWith("prop-999"); + expect(errors).toBeDefined(); + expect(errors![0].message).toBe( + 'must pass "proposalExists" keyword validation', + ); + }); + }); }); diff --git a/src/published-data/validator.service.ts b/src/published-data/validator.service.ts index 80550fc93..3db405018 100644 --- a/src/published-data/validator.service.ts +++ b/src/published-data/validator.service.ts @@ -5,8 +5,8 @@ import addKeywords from "ajv-keywords"; import def, { DynamicDefaultFunc, } from "ajv-keywords/dist/definitions/dynamicDefaults"; -import Ajv2019, { Schema } from "ajv/dist/2019"; -import { isArray, isEmpty, isMap, isNil } from "lodash"; +import Ajv2019, { KeywordDefinition, Schema } from "ajv/dist/2019"; +import { cloneDeep, isArray, isEmpty, isMap, isNil } from "lodash"; import { AttachmentsService } from "src/attachments/attachments.service"; import { DatasetsService } from "src/datasets/datasets.service"; import { ProposalsService } from "src/proposals/proposals.service"; @@ -30,10 +30,13 @@ export type ReadOnlyAttachmentsService = Pick< "findOne" | "findAll" | "count" >; +type Keyword = { keyword: string; validate: object }; + @Injectable() export class ValidatorService { private ajv: Ajv2019; private config: PublishedDataConfigDto; + private keywords: Keyword[] = []; private dynamicDefaults: Map = new Map([ ["currentYear", () => () => new Date().getFullYear()], ]); @@ -70,10 +73,7 @@ export class ValidatorService { const externalModule = this.loadExternalModule(modulePath!); if (isArray(externalModule.keywords)) { - for (const definition of externalModule.keywords) { - Logger.log(`Adding ajv keyword: '${definition.keyword}'`); - this.ajv.addKeyword(definition); - } + this.keywords = externalModule.keywords; } if (isMap(externalModule.dynamicDefaults)) { @@ -98,10 +98,11 @@ export class ValidatorService { return null; } - await this.loadDynamicDefaultFunctions(publishedData); + await this.loadDynamicFunctions(publishedData); const validateFn = this.ajv.compile(this.config.metadataSchema as Schema); validateFn(publishedData.metadata); + this.ajv.removeSchema(); return validateFn.errors; } @@ -113,46 +114,88 @@ export class ValidatorService { return externalModule; } - private async loadDynamicDefaultFunctions( + private async loadDynamicFunctions( publishedData: | CreatePublishedDataV4Dto | UpdatePublishedDataV4Dto | PartialUpdatePublishedDataV4Dto, ) { + const context = { + publishedData, + proposalService: this.proposalsService as ReadOnlyProposalsService, + datasetsService: this.datasetsService as ReadOnlyDatasetsService, + attachmentsService: this.attachmentsService as ReadOnlyAttachmentsService, + }; + for (const [name, implementation] of this.dynamicDefaults.entries()) { - if (typeof implementation !== "function") { - Logger.error( - `Ignoring dynamic defaults function ${name} should be of type 'function' not '${typeof implementation}'.`, - ); - continue; + const resolved = await this.resolveFunction( + implementation, + context, + name, + "dynamicDefaults function", + ); + if (!resolved) continue; + + def.DEFAULTS[name] = + implementation.constructor.name === "AsyncFunction" + ? () => resolved + : resolved; + } + + for (const keywordDefinition of this.keywords) { + const resolvedValidate = await this.resolveFunction( + keywordDefinition.validate, + context, + keywordDefinition.keyword, + "keyword", + ); + if (!resolvedValidate) continue; + + if (keywordDefinition.validate.constructor.name === "AsyncFunction") { + const k = cloneDeep(keywordDefinition); + k.validate = resolvedValidate; + this.overwriteKeyword(k); + } else { + this.overwriteKeyword(keywordDefinition); } - switch (implementation.constructor.name) { - case "Function": - def.DEFAULTS[name] = implementation; - break; - case "AsyncFunction": - /** - * Ajv cannot 'await' during validation. To get around this, we run the - * AsyncFunction now to perform any setup (like DB queries). - */ - try { - const syncFunc = await implementation({ - publishedData: publishedData, - proposalService: this - .proposalsService as ReadOnlyProposalsService, - datasetsService: this.datasetsService as ReadOnlyDatasetsService, - attachmentsService: this - .attachmentsService as ReadOnlyAttachmentsService, - }); - def.DEFAULTS[name] = () => syncFunc; - } catch (err) { - throw new Error( - `Executing dynamicDefaults function '${name}' failed with the following error:`, - { cause: err }, - ); - } - break; + } + } + + private async resolveFunction( + fn: unknown, + context: unknown, + name: string, + contextLabel: string, + ) { + if (typeof fn !== "function") { + Logger.error( + `Ignoring ${contextLabel} '${name}' should be of type 'function' not '${typeof fn}'.`, + ); + return null; + } + + if (fn.constructor.name === "AsyncFunction") { + try { + /** + * Ajv cannot 'await' during validation. To get around this, we run the + * AsyncFunction now to perform any setup (like DB queries). + */ + return await fn(context); + } catch (err) { + throw new Error( + `Executing ${contextLabel} '${name}' failed with the following error:`, + { cause: err }, + ); } } + + return fn; + } + + private overwriteKeyword(keywordDefinition: Keyword) { + if (this.ajv.getKeyword(keywordDefinition.keyword)) { + this.ajv.removeKeyword(keywordDefinition.keyword); + } + this.ajv.addKeyword(keywordDefinition as KeywordDefinition); } } From 9fba086dd4f8ace631ba1e6016711ce1e3887408 Mon Sep 17 00:00:00 2001 From: fpotier Date: Thu, 21 May 2026 13:51:52 +0000 Subject: [PATCH 2/2] address sourcery comments --- src/published-data/validator.service.ts | 145 +++++++++++++----------- 1 file changed, 76 insertions(+), 69 deletions(-) diff --git a/src/published-data/validator.service.ts b/src/published-data/validator.service.ts index 3db405018..4571fa0a6 100644 --- a/src/published-data/validator.service.ts +++ b/src/published-data/validator.service.ts @@ -6,7 +6,7 @@ import def, { DynamicDefaultFunc, } from "ajv-keywords/dist/definitions/dynamicDefaults"; import Ajv2019, { KeywordDefinition, Schema } from "ajv/dist/2019"; -import { cloneDeep, isArray, isEmpty, isMap, isNil } from "lodash"; +import { isArray, isEmpty, isMap, isNil } from "lodash"; import { AttachmentsService } from "src/attachments/attachments.service"; import { DatasetsService } from "src/datasets/datasets.service"; import { ProposalsService } from "src/proposals/proposals.service"; @@ -30,7 +30,17 @@ export type ReadOnlyAttachmentsService = Pick< "findOne" | "findAll" | "count" >; -type Keyword = { keyword: string; validate: object }; +export type ValidationContext = { + publishedData: + | CreatePublishedDataV4Dto + | UpdatePublishedDataV4Dto + | PartialUpdatePublishedDataV4Dto; + proposalService: ReadOnlyProposalsService; + datasetsService: ReadOnlyDatasetsService; + attachmentsService: ReadOnlyAttachmentsService; +}; + +type Keyword = { keyword: string; validate: unknown }; @Injectable() export class ValidatorService { @@ -47,14 +57,6 @@ export class ValidatorService { private readonly datasetsService: DatasetsService, private readonly attachmentsService: AttachmentsService, ) { - this.ajv = new Ajv2019({ - useDefaults: "empty", - allErrors: true, - strict: false, - }); - addFormats(this.ajv); - addKeywords(this.ajv); - this.config = this.configService.get( "publishedDataConfig", { metadataSchema: {}, uiSchema: {} }, @@ -98,11 +100,24 @@ export class ValidatorService { return null; } - await this.loadDynamicFunctions(publishedData); + this.ajv = new Ajv2019({ + useDefaults: "empty", + allErrors: true, + strict: false, + }); + addFormats(this.ajv); + addKeywords(this.ajv); + const context = { + publishedData, + proposalService: this.proposalsService as ReadOnlyProposalsService, + datasetsService: this.datasetsService as ReadOnlyDatasetsService, + attachmentsService: this.attachmentsService as ReadOnlyAttachmentsService, + }; + await this.loadDynamicDefaults(context); + await this.loadKeywords(context); const validateFn = this.ajv.compile(this.config.metadataSchema as Schema); validateFn(publishedData.metadata); - this.ajv.removeSchema(); return validateFn.errors; } @@ -110,92 +125,84 @@ export class ValidatorService { Logger.debug(`Loading custom ajv code at ${path}`); // eslint-disable-next-line @typescript-eslint/no-require-imports const externalModule = require(path); - return externalModule; } - private async loadDynamicFunctions( - publishedData: - | CreatePublishedDataV4Dto - | UpdatePublishedDataV4Dto - | PartialUpdatePublishedDataV4Dto, - ) { - const context = { - publishedData, - proposalService: this.proposalsService as ReadOnlyProposalsService, - datasetsService: this.datasetsService as ReadOnlyDatasetsService, - attachmentsService: this.attachmentsService as ReadOnlyAttachmentsService, - }; - + private async loadDynamicDefaults(context: ValidationContext) { for (const [name, implementation] of this.dynamicDefaults.entries()) { - const resolved = await this.resolveFunction( + const resolved = await this.resolveDynamicDefault( + name, implementation, context, - name, - "dynamicDefaults function", ); if (!resolved) continue; - - def.DEFAULTS[name] = - implementation.constructor.name === "AsyncFunction" - ? () => resolved - : resolved; + def.DEFAULTS[name] = resolved; } + } + private async loadKeywords(context: ValidationContext) { for (const keywordDefinition of this.keywords) { - const resolvedValidate = await this.resolveFunction( - keywordDefinition.validate, - context, - keywordDefinition.keyword, - "keyword", - ); - if (!resolvedValidate) continue; - - if (keywordDefinition.validate.constructor.name === "AsyncFunction") { - const k = cloneDeep(keywordDefinition); - k.validate = resolvedValidate; - this.overwriteKeyword(k); - } else { - this.overwriteKeyword(keywordDefinition); - } + const resolved = await this.resolveKeyword(keywordDefinition, context); + if (!resolved) continue; + this.ajv.addKeyword(resolved as KeywordDefinition); } } - private async resolveFunction( - fn: unknown, - context: unknown, + private async resolveDynamicDefault( name: string, - contextLabel: string, - ) { - if (typeof fn !== "function") { + implementation: unknown, + context: unknown, + ): Promise { + if (typeof implementation !== "function") { Logger.error( - `Ignoring ${contextLabel} '${name}' should be of type 'function' not '${typeof fn}'.`, + `Ignoring dynamicDefaults function '${name}' should be of type 'function' not '${typeof implementation}'.`, ); return null; } - if (fn.constructor.name === "AsyncFunction") { + if (implementation.constructor.name === "AsyncFunction") { try { - /** - * Ajv cannot 'await' during validation. To get around this, we run the - * AsyncFunction now to perform any setup (like DB queries). - */ - return await fn(context); + const syncFunc = await implementation(context); + return () => syncFunc; } catch (err) { throw new Error( - `Executing ${contextLabel} '${name}' failed with the following error:`, + `Executing dynamicDefaults function '${name}' failed with the following error:`, { cause: err }, ); } } - - return fn; + return implementation as DynamicDefaultFunc; } - private overwriteKeyword(keywordDefinition: Keyword) { - if (this.ajv.getKeyword(keywordDefinition.keyword)) { - this.ajv.removeKeyword(keywordDefinition.keyword); + private async resolveKeyword( + keywordDefinition: Keyword, + context: unknown, + ): Promise { + const { keyword, validate } = keywordDefinition; + + if (typeof validate !== "function") { + Logger.error( + `Ignoring keyword '${keyword}' validate should be of type 'function' not '${typeof validate}'.`, + ); + return null; } - this.ajv.addKeyword(keywordDefinition as KeywordDefinition); + + if (validate.constructor.name === "AsyncFunction") { + try { + const resolvedValidate = await validate(context); + const normalized: Keyword = { + ...keywordDefinition, + validate: resolvedValidate, + }; + return normalized; + } catch (err) { + throw new Error( + `Executing keyword '${keyword}' failed with the following error:`, + { cause: err }, + ); + } + } + + return keywordDefinition; } }