From 7694057cb185f0fa8ab4e6ddd6b2f615847978a1 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 10:29:06 +0200 Subject: [PATCH 01/10] feat: parse default-locale columns in translations spreadsheet Columns of the default (DB) locale now also yield the value of the object field itself, on top of the translation, so both stay in sync and the sheet round-trips through 'to-spreadsheet' (which reads locale columns from the translations). Locales are matched by language part, so "en" also matches a column mapped to "en_GB", and the field name comes from the column prefix ("Left side description" -> leftSideDescription). Co-Authored-By: Claude Opus 5 --- ...slationsRepositorySpreadsheetRepository.ts | 41 +++++++-- ...onsRepositorySpreadsheetRepository.spec.ts | 89 +++++++++++++++++++ src/domain/entities/FieldTranslations.ts | 7 +- src/domain/entities/Translation.ts | 17 ++++ .../ImportTranslationsRepository.ts | 5 +- 5 files changed, 150 insertions(+), 9 deletions(-) create mode 100644 src/data/__tests__/ImportTranslationsRepositorySpreadsheetRepository.spec.ts diff --git a/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts b/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts index 7f15b205..746417cb 100644 --- a/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts +++ b/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts @@ -1,6 +1,6 @@ import _ from "lodash"; import { Async } from "domain/entities/Async"; -import { FieldTranslation, FieldTranslations } from "domain/entities/FieldTranslations"; +import { FieldTranslation, FieldTranslations, FieldValues } from "domain/entities/FieldTranslations"; import { ImportTranslationsRepository, GetFieldTranslationsOptions, @@ -9,7 +9,8 @@ import { SpreadsheetXlsxDataSource } from "domain/repositories/SpreadsheetXlsxRe import log from "utils/log"; import { Maybe } from "utils/ts-utils"; import { getPluralModel } from "./dhis2-utils"; -import { translationFieldToProperty } from "domain/entities/Translation"; +import { Translation, translationFieldToProperty } from "domain/entities/Translation"; +import { getLocaleInfo, LocaleCode } from "domain/entities/Locale"; const columnsMapping = { id: ["id", "uid"], @@ -29,6 +30,9 @@ const columnsMapping = { - ... Spreadsheets starting with '!' will be skipped. + + Columns of the default locale (option --default-locale) also update the object field itself, + on top of its translation. */ export class ImportTranslationsRepositorySpreadsheetRepository implements ImportTranslationsRepository { async get(options: GetFieldTranslationsOptions): Async { @@ -38,11 +42,14 @@ export class ImportTranslationsRepositorySpreadsheetRepository implements Import return _(spreadsheet.sheets) .reject(sheet => sheet.name.startsWith("!")) .flatMap((sheet): FieldTranslations => { - return _(sheet.rows) + const fieldTranslations = _(sheet.rows) .map((row, rowIndex) => this.fromRow(row, rowIndex, `${sheet.name}:${rowIndex}`, options)) .compact() .value(); + log.info(`Sheet '${sheet.name}': ${fieldTranslations.length} rows parsed`); + return fieldTranslations; }) + .value(); } @@ -79,14 +86,14 @@ export class ImportTranslationsRepositorySpreadsheetRepository implements Import .filter(column => column.includes(":")) .value(); - const translations = _(translationColumns) + const entries = _(translationColumns) .flatMap(translationColumn => { const [fields = "", localeName] = translationColumn.split(":").map(s => s.trim()); return fields .split(",") .map(s => s.trim()) - .map(field => { + .map((field): Maybe => { const locale = localeName ? localesByName[localeName] : undefined; const text = row[translationColumn]; @@ -98,14 +105,22 @@ export class ImportTranslationsRepositorySpreadsheetRepository implements Import return undefined; } else { const property = translationFieldToProperty(field); - return { property: property, locale: locale.locale, value: text }; + const translation = { property: property, locale: locale.locale, value: text }; + // A default-locale column also updates the object field itself, so both + // the field and its translation stay in sync. + const isDefault = isDefaultLocale(locale.locale, options.defaultLocale); + const fieldValue = isDefault ? { [_.camelCase(field)]: text } : undefined; + return { translation, fieldValue }; } }); }) .compact() .value(); - return { model: getPluralModel(model), identifier: identifier, translations }; + const translations = entries.map(entry => entry.translation); + const fieldValues: FieldValues = Object.assign({}, ...entries.map(entry => entry.fieldValue)); + + return { model: getPluralModel(model), identifier: identifier, translations, fields: fieldValues }; } private getHeaderValue(row: Row, column: ColumnsMappingKey): Maybe { @@ -121,3 +136,15 @@ export class ImportTranslationsRepositorySpreadsheetRepository implements Import type Row = Record; type ColumnsMappingKey = keyof typeof columnsMapping; + +interface ColumnEntry { + translation: Translation; + fieldValue: Maybe; +} + +/* Locales may be LANGUAGE or LANGUAGE_COUNTRY: compare only the language part, so a + --default-locale=en also matches a column mapped to en_GB. */ +function isDefaultLocale(locale: LocaleCode, defaultLocale: Maybe): boolean { + if (!defaultLocale) return false; + return getLocaleInfo(locale).language === getLocaleInfo(defaultLocale).language; +} diff --git a/src/data/__tests__/ImportTranslationsRepositorySpreadsheetRepository.spec.ts b/src/data/__tests__/ImportTranslationsRepositorySpreadsheetRepository.spec.ts new file mode 100644 index 00000000..4f33ea3b --- /dev/null +++ b/src/data/__tests__/ImportTranslationsRepositorySpreadsheetRepository.spec.ts @@ -0,0 +1,89 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import * as XLSX from "xlsx"; +import { afterAll, describe, expect, test } from "vitest"; +import { ImportTranslationsRepositorySpreadsheetRepository } from "../ImportTranslationsRepositorySpreadsheetRepository"; +import { FieldTranslation } from "domain/entities/FieldTranslations"; +import { Locale } from "domain/entities/Locale"; +import { Maybe } from "utils/ts-utils"; + +const locales: Locale[] = [ + { id: "1", name: "English (United Kingdom)", locale: "en_GB" }, + { id: "2", name: "Spanish (Spain)", locale: "es" }, +]; + +const header = ["type", "id", "name: English", "formName: English", "formName: Spanish"]; +const row = ["dataElement", "abc", "Malaria cases", "Malaria form", "Formulario malaria"]; + +describe("ImportTranslationsRepositorySpreadsheetRepository", () => { + test("without a default locale, columns only produce translations", async () => { + const fieldTranslation = await getFirst(undefined); + + expect(fieldTranslation?.fields).toEqual({}); + expect(fieldTranslation?.translations).toEqual([ + { property: "NAME", locale: "en_GB", value: "Malaria cases" }, + { property: "FORM_NAME", locale: "en_GB", value: "Malaria form" }, + { property: "FORM_NAME", locale: "es", value: "Formulario malaria" }, + ]); + }); + + test("default-locale columns update the field and keep the translation", async () => { + // "en" matches the en_GB locale: only the language part is compared. + const fieldTranslation = await getFirst("en"); + + expect(fieldTranslation?.fields).toEqual({ name: "Malaria cases", formName: "Malaria form" }); + expect(fieldTranslation?.translations).toEqual([ + { property: "NAME", locale: "en_GB", value: "Malaria cases" }, + { property: "FORM_NAME", locale: "en_GB", value: "Malaria form" }, + { property: "FORM_NAME", locale: "es", value: "Formulario malaria" }, + ]); + }); + + test("columns of other locales are not written to the field", async () => { + const fieldTranslation = await getFirst("es"); + + expect(fieldTranslation?.fields).toEqual({ formName: "Formulario malaria" }); + }); + + test("spaced column names are converted to the object field name", async () => { + const fieldTranslation = await getFirst("en", { + header: ["type", "id", "Left side description: English"], + row: ["validationRule", "abc", "Left side"], + }); + + expect(fieldTranslation?.fields).toEqual({ leftSideDescription: "Left side" }); + expect(fieldTranslation?.translations).toEqual([ + { property: "LEFT_SIDE_DESCRIPTION", locale: "en_GB", value: "Left side" }, + ]); + }); +}); + +const tempDirs: string[] = []; + +afterAll(() => { + tempDirs.forEach(dir => fs.rmSync(dir, { recursive: true, force: true })); +}); + +async function getFirst( + defaultLocale: Maybe, + rows: { header: string[]; row: string[] } = { header, row } +): Promise> { + const inputFile = writeSpreadsheet([rows.header, rows.row]); + const repository = new ImportTranslationsRepositorySpreadsheetRepository(); + const fieldTranslations = await repository.get({ inputFile, locales, defaultLocale }); + + return fieldTranslations[0]; +} + +function writeSpreadsheet(rows: string[][]): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "d2-tools-translations-")); + tempDirs.push(dir); + + const inputFile = path.join(dir, "translations.xlsx"); + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet(rows), "Sheet1"); + XLSX.writeFile(workbook, inputFile); + + return inputFile; +} diff --git a/src/domain/entities/FieldTranslations.ts b/src/domain/entities/FieldTranslations.ts index 406c46d5..ffd673dc 100644 --- a/src/domain/entities/FieldTranslations.ts +++ b/src/domain/entities/FieldTranslations.ts @@ -1,4 +1,4 @@ -import { Translation } from "./Translation"; +import { TranslatableField, Translation } from "./Translation"; export type FieldTranslations = FieldTranslation[]; @@ -6,4 +6,9 @@ export interface FieldTranslation { model: string; // plural identifier: Partial<{ id: string; name: string; code: string }>; translations: Translation[]; + /* Values to write on the object itself, not as a translation. Filled from the columns whose + locale is the default locale (see option --default-locale). */ + fields: FieldValues; } + +export type FieldValues = Partial>; diff --git a/src/domain/entities/Translation.ts b/src/domain/entities/Translation.ts index 46c2eaf6..257c9df5 100644 --- a/src/domain/entities/Translation.ts +++ b/src/domain/entities/Translation.ts @@ -7,6 +7,23 @@ export interface Translation { value: string; } +/* A translatable field of a metadata object, as written in a spreadsheet column. The listed values + are the common ones (autocompleted by the editor); the set is open because translatable fields + vary per model (ex: executionDateLabel on programs, leftSideDescription on validation rules). */ +export type TranslatableField = + | "name" + | "shortName" + | "formName" + | "description" + | "leftSideDescription" + | "rightSideDescription" + | "executionDateLabel" + | (string & {}); + +/* Translatable fields whose value must be unique in the instance: writing them from a spreadsheet + may make the whole metadata payload fail to validate. */ +export const uniqueTranslatableFields: TranslatableField[] = ["name", "shortName"]; + /* Convert a spreadsheet field name into a DHIS2 translation property. Handles multi-word fields (more than one underscore) and both camelCase and spaced inputs. Examples: "formName" -> "FORM_NAME", "leftSideDescription" -> "LEFT_SIDE_DESCRIPTION". */ diff --git a/src/domain/repositories/ImportTranslationsRepository.ts b/src/domain/repositories/ImportTranslationsRepository.ts index bbdc34ec..d2205245 100644 --- a/src/domain/repositories/ImportTranslationsRepository.ts +++ b/src/domain/repositories/ImportTranslationsRepository.ts @@ -1,6 +1,6 @@ import { Async } from "domain/entities/Async"; import { FieldTranslations } from "domain/entities/FieldTranslations"; -import { Locale } from "domain/entities/Locale"; +import { Locale, LocaleCode } from "domain/entities/Locale"; export interface ImportTranslationsRepository { get(options: GetFieldTranslationsOptions): Async; @@ -9,4 +9,7 @@ export interface ImportTranslationsRepository { export interface GetFieldTranslationsOptions { inputFile: string; locales: Locale[]; + /* Columns of this locale are also written to the object field itself, not only as a + translation. Matched by language, so "en" also matches a column mapped to "en_GB". */ + defaultLocale?: LocaleCode; } From 6b86682d85173041456291e81e63105373841399 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 10:29:14 +0200 Subject: [PATCH 02/10] feat: add --default-locale to 'translations from-spreadsheet' Apply the default-locale values on the posted objects and warn about columns that would not do what the operator expects: - Fields the instance does not declare as translatable in /api/schemas: DHIS2 ignores unknown properties, so such a column posts fine but changes nothing. - Default-locale columns writing a unique field (name, shortName), where a duplicated value makes the whole payload fail to validate. Both are warnings (one per model/field, not per row), reviewed in the dry run that is still the default without --post. Co-Authored-By: Claude Opus 5 --- src/data/SchemasD2Repository.ts | 35 +++++ src/domain/repositories/SchemasRepository.ts | 8 ++ .../usecases/TranslateMetadataUseCase.ts | 48 ++++++- .../TranslateMetadataUseCase.spec.ts | 136 ++++++++++++++++++ src/scripts/commands/translations.ts | 10 ++ 5 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 src/data/SchemasD2Repository.ts create mode 100644 src/domain/repositories/SchemasRepository.ts create mode 100644 src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts diff --git a/src/data/SchemasD2Repository.ts b/src/data/SchemasD2Repository.ts new file mode 100644 index 00000000..d5e75e1b --- /dev/null +++ b/src/data/SchemasD2Repository.ts @@ -0,0 +1,35 @@ +import _ from "lodash"; +import { Async } from "domain/entities/Async"; +import { MetadataModel } from "domain/entities/MetadataObject"; +import { TranslatableField } from "domain/entities/Translation"; +import { SchemasRepository } from "domain/repositories/SchemasRepository"; +import { D2Api } from "types/d2-api"; + +export class SchemasD2Repository implements SchemasRepository { + constructor(private api: D2Api) {} + + async getTranslatableFields(): Async> { + const res = await this.api + .get("/schemas", { fields: "plural,properties[name,translatable]" }) + .getData(); + + return _(res.schemas) + .map(schema => { + const fields = _(schema.properties) + .filter(property => property.translatable) + .map(property => property.name) + .value(); + + return [schema.plural, fields] as [MetadataModel, TranslatableField[]]; + }) + .fromPairs() + .value(); + } +} + +interface D2SchemasResponse { + schemas: Array<{ + plural: string; + properties: Array<{ name: string; translatable: boolean }>; + }>; +} diff --git a/src/domain/repositories/SchemasRepository.ts b/src/domain/repositories/SchemasRepository.ts new file mode 100644 index 00000000..53cc5552 --- /dev/null +++ b/src/domain/repositories/SchemasRepository.ts @@ -0,0 +1,8 @@ +import { Async } from "domain/entities/Async"; +import { MetadataModel } from "domain/entities/MetadataObject"; +import { TranslatableField } from "domain/entities/Translation"; + +export interface SchemasRepository { + /* Translatable fields of each model, keyed by plural model name. */ + getTranslatableFields(): Async>; +} diff --git a/src/domain/usecases/TranslateMetadataUseCase.ts b/src/domain/usecases/TranslateMetadataUseCase.ts index 12b4d6e3..70573f69 100644 --- a/src/domain/usecases/TranslateMetadataUseCase.ts +++ b/src/domain/usecases/TranslateMetadataUseCase.ts @@ -3,18 +3,21 @@ import fs from "fs"; import { Async } from "domain/entities/Async"; import { MetadataRepository } from "domain/repositories/MetadataRepository"; import { ImportTranslationsRepository } from "domain/repositories/ImportTranslationsRepository"; -import { Translation } from "domain/entities/Translation"; +import { TranslatableField, Translation, uniqueTranslatableFields } from "domain/entities/Translation"; import log from "utils/log"; import { FieldTranslations } from "domain/entities/FieldTranslations"; import { LocalesRepository } from "domain/repositories/LocalesRepository"; +import { SchemasRepository } from "domain/repositories/SchemasRepository"; import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; import { Maybe } from "utils/ts-utils"; import { getId } from "domain/entities/Base"; +import { LocaleCode } from "domain/entities/Locale"; interface Options { inputFile: string; savePayload?: string; post: boolean; + defaultLocale: Maybe; } export class TranslateMetadataUseCase { @@ -22,6 +25,7 @@ export class TranslateMetadataUseCase { private repositories: { metadata: MetadataRepository; locales: LocalesRepository; + schemas: SchemasRepository; importTranslations: ImportTranslationsRepository; } ) {} @@ -29,7 +33,7 @@ export class TranslateMetadataUseCase { async execute(options: Options): Async { const { savePayload: saveToFile } = options; const objectsToPost = await this.getObjectsToPost(options); - log.info(`Payload: ${objectsToPost.length} objects`); + log.info(`Objects with changes: ${objectsToPost.length}`); const dryRun = !options.post; const { stats, payload } = await this.repositories.metadata.save(objectsToPost, { dryRun }); @@ -49,8 +53,11 @@ export class TranslateMetadataUseCase { const fieldTranslations = await this.repositories.importTranslations.get({ inputFile: options.inputFile, locales: locales, + defaultLocale: options.defaultLocale, }); + await this.validateFields(fieldTranslations); + const models = _(fieldTranslations) .map(o => o.model) .uniq() @@ -59,10 +66,46 @@ export class TranslateMetadataUseCase { const objects = await this.repositories.metadata.getAllWithTranslations(models); const objectsWithTranslations = this.addTranslations(objects, fieldTranslations); const objectsWithChanges = _.differenceWith(objectsWithTranslations, objects, _.isEqual); + log.info(`Objects with translations: ${objectsWithTranslations.length}`); return objectsWithChanges; } + /* Warn about columns whose field the instance does not consider translatable (DHIS2 ignores + unknown properties, so those columns would silently do nothing) and about default-locale + columns writing unique-constrained fields. One warning per model/field, not per row. */ + private async validateFields(fieldTranslations: FieldTranslations): Async { + const translatableFieldsByModel = await this.repositories.schemas.getTranslatableFields(); + + const usages = _(fieldTranslations) + .flatMap(({ model, fields, translations }) => { + const fromFields = _.keys(fields).map(field => ({ model, field, isFieldValue: true })); + // "FORM_NAME" -> "formName", back to the field the column refers to. + const fromTranslations = translations.map(translation => ({ + model: model, + field: _.camelCase(translation.property), + isFieldValue: false, + })); + return [...fromFields, ...fromTranslations]; + }) + .uniqBy(usage => [usage.model, usage.field, usage.isFieldValue].join(".")) + .value(); + + usages.forEach(({ model, field, isFieldValue }) => { + // An unknown model is already reported by the object lookup, don't warn twice. + const translatableFields: Maybe = translatableFieldsByModel[model]; + + if (translatableFields && !translatableFields.includes(field)) { + log.warn(`Field not translatable, column ignored: ${model}.${field}`); + } else if (isFieldValue && uniqueTranslatableFields.includes(field)) { + log.warn( + `Default locale writes the unique field ${model}.${field}: ` + + `duplicated values will make the import fail` + ); + } + }); + } + private addTranslations( objects: MetadataObjectWithTranslations[], fieldTranslations: FieldTranslations @@ -89,6 +132,7 @@ export class TranslateMetadataUseCase { } else { return { ...object, + ...fieldTranslation.fields, translations: this.mergeTranslations( object.translations, fieldTranslation.translations diff --git a/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts b/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts new file mode 100644 index 00000000..439bdc1a --- /dev/null +++ b/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts @@ -0,0 +1,136 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { TranslateMetadataUseCase } from "../TranslateMetadataUseCase"; +import { FieldTranslation } from "domain/entities/FieldTranslations"; +import { Locale } from "domain/entities/Locale"; +import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; +import { MetadataRepository } from "domain/repositories/MetadataRepository"; +import { LocalesRepository } from "domain/repositories/LocalesRepository"; +import { SchemasRepository } from "domain/repositories/SchemasRepository"; +import { ImportTranslationsRepository } from "domain/repositories/ImportTranslationsRepository"; +import log from "utils/log"; + +const locales: Locale[] = [ + { id: "1", name: "English (United Kingdom)", locale: "en_GB" }, + { id: "2", name: "Spanish (Spain)", locale: "es" }, +]; + +const object: MetadataObjectWithTranslations = { + model: "dataElements", + id: "abc", + name: "Malaria cases", + code: "MAL", + translations: [{ property: "FORM_NAME", locale: "es", value: "Formulario antiguo" }], +}; + +const fieldTranslation: FieldTranslation = { + model: "dataElements", + identifier: { id: "abc" }, + translations: [ + { property: "FORM_NAME", locale: "en_GB", value: "Malaria form" }, + { property: "FORM_NAME", locale: "es", value: "Formulario malaria" }, + ], + fields: { formName: "Malaria form" }, +}; + +describe("TranslateMetadataUseCase", () => { + beforeEach(() => { + vi.spyOn(log, "warn").mockImplementation(() => undefined); + }); + + test("passes the default locale to the spreadsheet reader", async () => { + const { useCase, importTranslations } = buildUseCase(); + + await useCase.execute({ inputFile: "in.xlsx", post: false, defaultLocale: "en" }); + + expect(importTranslations.get).toHaveBeenCalledWith( + expect.objectContaining({ inputFile: "in.xlsx", defaultLocale: "en" }) + ); + }); + + test("writes the default-locale fields on the object, on top of the translations", async () => { + const { useCase, metadata } = buildUseCase(); + + await useCase.execute({ inputFile: "in.xlsx", post: true, defaultLocale: "en" }); + + const [objects] = metadata.save.mock.calls[0]; + expect(objects).toEqual([ + { + ...object, + formName: "Malaria form", + translations: [ + { property: "FORM_NAME", locale: "es", value: "Formulario malaria" }, + { property: "FORM_NAME", locale: "en_GB", value: "Malaria form" }, + ], + }, + ]); + }); + + test("warns about fields the instance does not consider translatable", async () => { + const { useCase } = buildUseCase({ + fieldTranslations: [ + { ...fieldTranslation, translations: [], fields: { fromName: "Typo'd column" } }, + ], + }); + + await useCase.execute({ inputFile: "in.xlsx", post: false, defaultLocale: "en" }); + + expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("dataElements.fromName")); + }); + + test("warns when the default locale writes a unique field", async () => { + const { useCase } = buildUseCase({ + fieldTranslations: [ + { ...fieldTranslation, translations: [], fields: { name: "Malaria cases (new)" } }, + ], + }); + + await useCase.execute({ inputFile: "in.xlsx", post: false, defaultLocale: "en" }); + + expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("unique field dataElements.name")); + }); + + test("does not warn about unique fields when they are only translated", async () => { + const { useCase } = buildUseCase({ + fieldTranslations: [ + { + ...fieldTranslation, + translations: [{ property: "NAME", locale: "es", value: "Casos de malaria" }], + fields: {}, + }, + ], + }); + + await useCase.execute({ inputFile: "in.xlsx", post: false, defaultLocale: undefined }); + + expect(log.warn).not.toHaveBeenCalled(); + }); +}); + +function buildUseCase(options: { fieldTranslations?: FieldTranslation[] } = {}) { + const metadata = { + getAllWithTranslations: vi.fn().mockResolvedValue([object]), + getPaginated: vi.fn(), + save: vi.fn().mockResolvedValue({ payload: {}, stats: {} }), + }; + + const localesRepo: LocalesRepository = { get: vi.fn().mockResolvedValue(locales) }; + + const schemas: SchemasRepository = { + getTranslatableFields: vi.fn().mockResolvedValue({ + dataElements: ["name", "shortName", "formName", "description"], + }), + }; + + const importTranslations = { + get: vi.fn().mockResolvedValue(options.fieldTranslations ?? [fieldTranslation]), + }; + + const useCase = new TranslateMetadataUseCase({ + metadata: metadata as unknown as MetadataRepository, + locales: localesRepo, + schemas: schemas, + importTranslations: importTranslations as unknown as ImportTranslationsRepository, + }); + + return { useCase, metadata, localesRepo, schemas, importTranslations }; +} diff --git a/src/scripts/commands/translations.ts b/src/scripts/commands/translations.ts index 9bdde295..628eebcd 100644 --- a/src/scripts/commands/translations.ts +++ b/src/scripts/commands/translations.ts @@ -8,6 +8,7 @@ import { LocalesD2Repository } from "data/LocalesD2Repository"; import { ImportTranslationsRepositorySpreadsheetRepository } from "data/ImportTranslationsRepositorySpreadsheetRepository"; import { ExportTranslationsSpreadsheetRepository } from "data/ExportTranslationsSpreadsheetRepository"; import { MetadataD2Repository } from "data/MetadataD2Repository"; +import { SchemasD2Repository } from "data/SchemasD2Repository"; export function getCommand() { const translateFromSpreadsheetCmd = command({ @@ -24,6 +25,14 @@ export function getCommand() { long: "save-payload", description: "Save JSON payload to file", }), + defaultLocale: option({ + type: optional(string), + long: "default-locale", + description: + "Locale code of the default (DB) language. Its columns update the object field " + + "itself, on top of its translation. Matched by language, so 'en' also matches " + + "a locale en_GB. Example: en", + }), inputFile: positional({ type: string, displayName: "INPUT_XLSX_PATH", @@ -36,6 +45,7 @@ export function getCommand() { const repositories = { metadata: new MetadataD2Repository(api), locales: new LocalesD2Repository(api), + schemas: new SchemasD2Repository(api), importTranslations: new ImportTranslationsRepositorySpreadsheetRepository(), }; await new TranslateMetadataUseCase(repositories).execute(args); From 9e2dc751b7532c40e5bec66033b3799cb85c6e24 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 10:29:20 +0200 Subject: [PATCH 03/10] docs: document --default-locale in 'translations from-spreadsheet' Co-Authored-By: Claude Opus 5 --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 1f915109..fe049399 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,7 @@ Update objects from spreadsheet. Update any type of DHIS2 metadata object using $ yarn start translations from-spreadsheet \ --url='http://USER:PASSWORD@HOST:PORT' \ --save-payload=payload.json \ + --default-locale=en \ --post \ translations.xlsx ``` @@ -269,6 +270,11 @@ Expected format of `xlsx` file: - Columns named `id`/`name`/`code` will be used to match the existing object in the database. No need to specify all of them. - Translation columns should have the format: `field:localeName`. A DHIS2 Locale with that name should exist in the database. Example: `formName:French`. +Notes: + +- `--default-locale`: locale code of the default (DB) language, `en` for example. Its columns update the object field itself (`formName: English` writes `formName`) on top of creating the translation, so both stay in sync. The match uses only the language part, so `en` also matches a locale `en_GB`. +- A warning is shown for columns whose field the instance does not consider translatable (DHIS2 ignores unknown properties, so those columns would silently do nothing), and when `--default-locale` writes a unique field such as `name` or `shortName`, since duplicated values make the whole import fail. + ### To spreadsheet Generate a translations spreadsheet from the metadata objects of a DHIS2 instance. The output is re-importable by `from-spreadsheet`: From 455f01461f415ebefe98b2d3cce2d27e6fb6fe5e Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 10:29:20 +0200 Subject: [PATCH 04/10] docs(openspec): add import-translations-from-spreadsheet spec Capability spec for the whole 'translations from-spreadsheet' command, not just the default-locale addition: spreadsheet format, object matching, translation merging, schema validation warnings and the dry-run default. Co-Authored-By: Claude Opus 5 --- .../spec.md | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 openspec/specs/import-translations-from-spreadsheet/spec.md diff --git a/openspec/specs/import-translations-from-spreadsheet/spec.md b/openspec/specs/import-translations-from-spreadsheet/spec.md new file mode 100644 index 00000000..68190e64 --- /dev/null +++ b/openspec/specs/import-translations-from-spreadsheet/spec.md @@ -0,0 +1,102 @@ +# import-translations-from-spreadsheet Specification + +## Purpose + +Bulk-update the translations of any DHIS2 metadata model from an xlsx spreadsheet, so translators +work in a familiar tool and their output is applied without manual editing in the DHIS2 UI. The +counterpart command `translations to-spreadsheet` generates a re-importable file of this format. + +## Requirements + +### Requirement: Read translations from an xlsx spreadsheet + +The `translations from-spreadsheet` command SHALL take an input xlsx path and parse every sheet in +it, except sheets whose name starts with `!`. Each row describes one metadata object. + +A row SHALL declare its model in a `type`/`kind` column, holding the singular model name (e.g. +`dataElement`), which the command pluralizes to address the DHIS2 API. Translation columns SHALL +have the format `: ` (e.g. `formName: French`); one column MAY list several +comma-separated fields sharing a value. Column matching is case-insensitive for the metadata +columns, and locale names are matched ignoring any ` (...)` suffix, so `Spanish` matches a +`Spanish (Spain)` DB locale. + +Rows without a model, without any identifier, or with an empty cell SHALL be skipped with a +warning rather than aborting the run: a partially filled translation sheet is the normal case. + +#### Scenario: Locale column not defined in the instance + +- **WHEN** a column names a locale that does not exist in `/api/locales/dbLocales` +- **THEN** the column is skipped and a warning naming the locale is logged once for the sheet + +### Requirement: Match objects by id, code or name + +The command SHALL locate the object each row refers to using the `id`/`uid`, `code` and `name` +columns, in that order of precedence, within the row's model. A row need only provide one of them. +Name matching SHALL be case-insensitive. A row matching no object SHALL be reported and skipped. + +#### Scenario: Unmatched row does not abort the run + +- **WHEN** a row's identifier matches no object of its model +- **THEN** a warning naming the model and identifier is logged, the row is skipped, and the + remaining rows are still imported + +### Requirement: Merge translations, never drop existing ones + +The command SHALL merge the spreadsheet translations into the object's existing ones, keyed by +(locale, property): a spreadsheet value overwrites the same locale/property, and any translation +absent from the spreadsheet SHALL be preserved. Only objects whose resulting payload differs from +the current one SHALL be posted. + +#### Scenario: Untouched locale survives the import + +- **WHEN** an object has a French `FORM_NAME` translation and the sheet only carries a Spanish one +- **THEN** the posted object keeps the French translation and gains the Spanish one + +### Requirement: Default-locale columns also update the object field + +The command SHALL provide a `--default-locale` option taking the locale code of the instance's +default (DB) language. Columns of that locale SHALL update the object's own field (`formName: +English` writes `formName`) **in addition to** writing the translation, so field and translation +stay in sync and the sheet round-trips through `to-spreadsheet`, which reads the locale columns +from the translations. + +The field name SHALL be derived from the column prefix (`Left side description` → +`leftSideDescription`). Locale comparison SHALL use only the language part, so `en` matches a +`en_GB` DB locale. Without the option, no object field is written. + +#### Scenario: Default-locale column writes both + +- **WHEN** the sheet has a `formName: English` column, English is the `en_GB` DB locale and the + command runs with `--default-locale=en` +- **THEN** the posted object has both `formName` set and an `en_GB` `FORM_NAME` translation + +### Requirement: Warn about fields that would silently do nothing or break the import + +The command SHALL check every column's field against the translatable properties the instance +declares in `/api/schemas`, and SHALL warn once per model/field (not per row) when the field is +not translatable there — DHIS2 ignores unknown properties, so such a column would post +successfully while changing nothing. + +It SHALL also warn when `--default-locale` writes a unique-constrained field (`name`, +`shortName`), because a duplicated value makes the whole metadata payload fail to validate. + +Both cases are warnings, not errors: the operator reviews them in the dry run before posting. + +#### Scenario: Typo in a column field name + +- **WHEN** a column is headed `fromName: English` and the instance declares `formName` (not + `fromName`) as translatable for that model +- **THEN** a warning naming the model and field is logged once, and the run continues + +### Requirement: Dry-run by default, apply with --post + +The command SHALL only persist changes when `--post` is given; otherwise it SHALL validate the +payload against the instance (`importMode=VALIDATE`) and report what would change. The +`--save-payload` option SHALL write the computed metadata payload to a JSON file in either mode, +so it can be reviewed or replayed. + +#### Scenario: Dry run reports without writing + +- **WHEN** the command runs without `--post` +- **THEN** no metadata is modified, the import stats are reported, and the operator is told to add + `--post` to persist From 63635dbfd4db135b69b5a462f5fd403ee689bb55 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 11:10:25 +0200 Subject: [PATCH 05/10] feat: add version to DataSet/Program and a programs save The Capture apps cache the metadata of each data set/program and refresh it only when its version changes, so the version has to be readable and writable to invalidate that cache. ProgramsRepository had no save. The new one merges the changed fields over the stored :owner object instead of posting the entity as it is: the entity is a partial view of the D2 model and the metadata import uses mergeMode=REPLACE, which would drop every field missing from the payload (in particular turning the nested programStages into partial objects). Same approach as MetadataD2Repository.mergeWithExistingObjects. Co-Authored-By: Claude Opus 5 --- src/data/DataSetsD2Repository.ts | 1 + src/data/ProgramsD2Repository.ts | 48 ++++++++++++- .../__tests__/ProgramsD2Repository.spec.ts | 68 +++++++++++++++++++ src/domain/entities/DataSet.ts | 3 + src/domain/entities/Program.ts | 2 + src/domain/repositories/ProgramsRepository.ts | 1 + 6 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 src/data/__tests__/ProgramsD2Repository.spec.ts diff --git a/src/data/DataSetsD2Repository.ts b/src/data/DataSetsD2Repository.ts index 19cbefd2..4aa40652 100644 --- a/src/data/DataSetsD2Repository.ts +++ b/src/data/DataSetsD2Repository.ts @@ -109,6 +109,7 @@ const fields = { id: true, name: true, code: true, + version: true, categoryCombo: { id: true, categoryOptionCombos: { id: true, name: true } }, dataSetElements: { dataElement: { diff --git a/src/data/ProgramsD2Repository.ts b/src/data/ProgramsD2Repository.ts index 12ca0be2..621ee7e2 100644 --- a/src/data/ProgramsD2Repository.ts +++ b/src/data/ProgramsD2Repository.ts @@ -1,6 +1,6 @@ import _ from "lodash"; import { Async } from "domain/entities/Async"; -import { Id } from "domain/entities/Base"; +import { getId, Id } from "domain/entities/Base"; import { ProgramExport } from "domain/entities/ProgramExport"; import { ProgramsRepository, RunRulesOptions } from "domain/repositories/ProgramsRepository"; import { @@ -32,6 +32,7 @@ export class ProgramsD2Repository implements ProgramsRepository { id: true, name: true, programType: true, + version: true, programStages: { id: true, name: true, @@ -59,6 +60,24 @@ export class ProgramsD2Repository implements ProgramsRepository { return programs; } + async save(programs: Program[]): Async { + // The Program entity is a partial view of the D2 model, so the changed fields must be + // merged over the stored object: the metadata import uses mergeMode=REPLACE and would + // drop everything missing from the payload. + const { programs: programsExisting } = await this.api.metadata + .get({ + programs: { + fields: { $owner: true }, + filter: { id: { in: programs.map(getId) } }, + }, + }) + .getData(); + + const payload: object = { programs: mergeProgramsWithExisting(programsExisting, programs) }; + const res = await runMetadata(this.api.metadata.post(payload)); + log.info(`Save programs (${programs.length}): ${res.status}`); + } + async export(options: { ids: Id[]; orgUnitIds: Id[] | undefined }): Async { const { ids: programIds, orgUnitIds } = options; const metadata = await this.getMetadata(programIds); @@ -138,6 +157,33 @@ export class ProgramsD2Repository implements ProgramsRepository { } } +/* Fields the Program entity owns and may write back. Everything else in the payload (including + the nested collections, which the entity holds only as a partial view) comes from the stored + object, as returned by the :owner preset. */ +const programWritableFields = ["name", "programType", "version"] as const; + +export function mergeProgramsWithExisting(programsExisting: D2ProgramOwner[], programs: Program[]): object[] { + const existingById = _.keyBy(programsExisting, program => program.id); + + return _(programs) + .map(program => { + const programExisting = existingById[program.id]; + + if (!programExisting) { + log.warn(`Cannot find program: ${program.id}`); + return undefined; + } else { + return { ...programExisting, ..._.pick(program, programWritableFields) }; + } + }) + .compact() + .value(); +} + +export interface D2ProgramOwner { + id: Id; +} + interface D2ProgramExport { metadata: object; data: D2ProgramData; diff --git a/src/data/__tests__/ProgramsD2Repository.spec.ts b/src/data/__tests__/ProgramsD2Repository.spec.ts new file mode 100644 index 00000000..b868e109 --- /dev/null +++ b/src/data/__tests__/ProgramsD2Repository.spec.ts @@ -0,0 +1,68 @@ +import { describe, expect, test, vi } from "vitest"; +import { mergeProgramsWithExisting } from "../ProgramsD2Repository"; +import { Program } from "domain/entities/Program"; +import log from "utils/log"; + +/* The :owner representation, as stored: nested collections are plain id-refs and there are many + fields the Program entity does not know about. */ +const programExisting = { + id: "PR1", + name: "Malaria program", + programType: "WITH_REGISTRATION", + version: 3, + shortName: "Malaria", + programStages: [{ id: "stage1" }, { id: "stage2" }], + programTrackedEntityAttributes: [{ id: "ptea1" }], +}; + +const program: Program = { + id: "PR1", + name: "Malaria program", + programType: "WITH_REGISTRATION", + version: 4, + programStages: [ + { + id: "stage1", + programStageDataElements: [ + { + dataElement: { + id: "abc", + name: "Element", + code: "DE", + valueType: "TEXT", + optionSet: undefined, + }, + displayInReports: false, + }, + ], + }, + ], +}; + +describe("mergeProgramsWithExisting", () => { + test("takes the writable fields from the entity", () => { + const [merged] = mergeProgramsWithExisting([programExisting], [program]); + + expect(merged).toMatchObject({ version: 4, name: "Malaria program" }); + }); + + test("keeps every other field of the stored object, nested collections included", () => { + const [merged] = mergeProgramsWithExisting([programExisting], [program]); + + // The entity only holds a partial view of programStages: it must not reach the payload. + expect(merged).toMatchObject({ + shortName: "Malaria", + programStages: [{ id: "stage1" }, { id: "stage2" }], + programTrackedEntityAttributes: [{ id: "ptea1" }], + }); + }); + + test("skips programs with no stored object", () => { + const warn = vi.spyOn(log, "warn").mockImplementation(() => undefined); + + const merged = mergeProgramsWithExisting([], [program]); + + expect(merged).toEqual([]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("PR1")); + }); +}); diff --git a/src/domain/entities/DataSet.ts b/src/domain/entities/DataSet.ts index dd907f70..cb5c2424 100644 --- a/src/domain/entities/DataSet.ts +++ b/src/domain/entities/DataSet.ts @@ -1,3 +1,4 @@ +import { Maybe } from "utils/ts-utils"; import { Id, NamedRef, Ref } from "./Base"; export interface DataSetToCompare { @@ -47,6 +48,8 @@ export interface DataSet { id: Id; name: string; code: string; + // The Capture apps cache the data set metadata and refresh it only when the version changes. + version: Maybe; skipOffline: boolean; categoryCombo: CategoryCombo; dataSetElements: Array<{ diff --git a/src/domain/entities/Program.ts b/src/domain/entities/Program.ts index e63e265b..f56c42d1 100644 --- a/src/domain/entities/Program.ts +++ b/src/domain/entities/Program.ts @@ -7,6 +7,8 @@ export interface Program { id: Id; name: string; programType: ProgramType; + // The Capture apps cache the program metadata and refresh it only when the version changes. + version: Maybe; programStages: ProgramStage[]; } diff --git a/src/domain/repositories/ProgramsRepository.ts b/src/domain/repositories/ProgramsRepository.ts index f9556c33..893ab938 100644 --- a/src/domain/repositories/ProgramsRepository.ts +++ b/src/domain/repositories/ProgramsRepository.ts @@ -6,6 +6,7 @@ import { ProgramExport } from "domain/entities/ProgramExport"; export interface ProgramsRepository { get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async; + save(programs: Program[]): Async; export(options: { ids: Id[] }): Async; import(programExport: ProgramExport): Async; runRules(options: RunRulesOptions): Async; From 19bf6caa36ad4bd5379a7e6bc88c85166a5b3884 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 11:10:34 +0200 Subject: [PATCH 06/10] feat: add --bump-versions to 'translations from-spreadsheet' Increment the version of the data sets and programs using a data element whose translations changed, so the Capture apps refresh their cached metadata instead of keeping showing the old values. Opt-in, as it writes objects the operator never listed in the spreadsheet. The scope is data elements only: translating an option set or a tracked entity attribute does not bump anything, even though those are cached too. Bumps are posted apart from the translations payload, so --save-payload does not show them, and a dry run only logs them. Co-Authored-By: Claude Opus 5 --- .../usecases/TranslateMetadataUseCase.ts | 63 +++++++- .../TranslateMetadataUseCase.spec.ts | 144 +++++++++++++++++- src/scripts/commands/translations.ts | 10 ++ 3 files changed, 210 insertions(+), 7 deletions(-) diff --git a/src/domain/usecases/TranslateMetadataUseCase.ts b/src/domain/usecases/TranslateMetadataUseCase.ts index 70573f69..5321410e 100644 --- a/src/domain/usecases/TranslateMetadataUseCase.ts +++ b/src/domain/usecases/TranslateMetadataUseCase.ts @@ -8,9 +8,11 @@ import log from "utils/log"; import { FieldTranslations } from "domain/entities/FieldTranslations"; import { LocalesRepository } from "domain/repositories/LocalesRepository"; import { SchemasRepository } from "domain/repositories/SchemasRepository"; +import { DataSetsRepository } from "domain/repositories/DataSetsRepository"; +import { ProgramsRepository } from "domain/repositories/ProgramsRepository"; import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; import { Maybe } from "utils/ts-utils"; -import { getId } from "domain/entities/Base"; +import { getId, Id } from "domain/entities/Base"; import { LocaleCode } from "domain/entities/Locale"; interface Options { @@ -18,6 +20,7 @@ interface Options { savePayload?: string; post: boolean; defaultLocale: Maybe; + bumpVersions: boolean; } export class TranslateMetadataUseCase { @@ -27,6 +30,8 @@ export class TranslateMetadataUseCase { locales: LocalesRepository; schemas: SchemasRepository; importTranslations: ImportTranslationsRepository; + dataSets: DataSetsRepository; + programs: ProgramsRepository; } ) {} @@ -45,6 +50,51 @@ export class TranslateMetadataUseCase { const contents = JSON.stringify(payload, null, 4); fs.writeFileSync(saveToFile, contents); } + + if (options.bumpVersions) await this.bumpVersions(objectsToPost, options); + } + + /* The Capture apps cache the metadata of each data set/program and refresh it only when its + version changes, so a translation update stays invisible until the data sets/programs using + the changed data elements are bumped. Posted separately, so --save-payload does not show it. */ + private async bumpVersions(objects: MetadataObjectWithTranslations[], options: Options): Async { + const dataElementIds = new Set(objects.filter(object => object.model === "dataElements").map(getId)); + + if (_.isEmpty(dataElementIds)) return; + + const [dataSets, programs] = await Promise.all([ + this.repositories.dataSets.getAll(), + this.repositories.programs.get({}), + ]); + + const dataSetsToBump = dataSets.filter(dataSet => + dataSet.dataSetElements.some(({ dataElement }) => dataElementIds.has(dataElement.id)) + ); + + const programsToBump = programs.filter(program => + program.programStages.some(programStage => + programStage.programStageDataElements.some(({ dataElement }) => + dataElementIds.has(dataElement.id) + ) + ) + ); + + const dataSetsBumped = dataSetsToBump.map(dataSet => bumpVersion(dataSet, "dataSets")); + const programsBumped = programsToBump.map(program => bumpVersion(program, "programs")); + + if (!options.post) { + log.info(`Versions not bumped (dryRun=true). Add option --post to persist`); + return; + } + + if (!_.isEmpty(dataSetsBumped)) { + const result = await this.repositories.dataSets.post({ dataSets: dataSetsBumped }); + if (result === "ERROR") throw new Error("Error while posting the dataSets"); + } + + if (!_.isEmpty(programsBumped)) { + await this.repositories.programs.save(programsBumped); + } } private async getObjectsToPost(options: Options) { @@ -161,3 +211,14 @@ export class TranslateMetadataUseCase { .value(); } } + +/* Objects never versioned have no version, start them at 1. */ +function bumpVersion }>( + object: Obj, + model: string +): Obj { + const version = (object.version ?? 0) + 1; + log.info(`Bump version ${model}:${object.id} (${object.name}): ${object.version ?? "-"} -> ${version}`); + + return { ...object, version }; +} diff --git a/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts b/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts index 439bdc1a..d686a181 100644 --- a/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts +++ b/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts @@ -7,7 +7,12 @@ import { MetadataRepository } from "domain/repositories/MetadataRepository"; import { LocalesRepository } from "domain/repositories/LocalesRepository"; import { SchemasRepository } from "domain/repositories/SchemasRepository"; import { ImportTranslationsRepository } from "domain/repositories/ImportTranslationsRepository"; +import { DataSetsRepository } from "domain/repositories/DataSetsRepository"; +import { ProgramsRepository } from "domain/repositories/ProgramsRepository"; +import { DataSet } from "domain/entities/DataSet"; +import { Program } from "domain/entities/Program"; import log from "utils/log"; +import { Maybe } from "utils/ts-utils"; const locales: Locale[] = [ { id: "1", name: "English (United Kingdom)", locale: "en_GB" }, @@ -22,6 +27,12 @@ const object: MetadataObjectWithTranslations = { translations: [{ property: "FORM_NAME", locale: "es", value: "Formulario antiguo" }], }; +const categoryCombo = { id: "cc", categoryOptionCombos: [] }; + +/* One data set/program per data element, so a lookup that ignores the references shows up. */ +const allDataSets: DataSet[] = [buildDataSet("DS1", "abc", 3), buildDataSet("DS2", "xyz", 1)]; +const allPrograms: Program[] = [buildProgram("PR1", "abc", undefined), buildProgram("PR2", "xyz", 5)]; + const fieldTranslation: FieldTranslation = { model: "dataElements", identifier: { id: "abc" }, @@ -40,7 +51,12 @@ describe("TranslateMetadataUseCase", () => { test("passes the default locale to the spreadsheet reader", async () => { const { useCase, importTranslations } = buildUseCase(); - await useCase.execute({ inputFile: "in.xlsx", post: false, defaultLocale: "en" }); + await useCase.execute({ + inputFile: "in.xlsx", + post: false, + defaultLocale: "en", + bumpVersions: false, + }); expect(importTranslations.get).toHaveBeenCalledWith( expect.objectContaining({ inputFile: "in.xlsx", defaultLocale: "en" }) @@ -50,7 +66,7 @@ describe("TranslateMetadataUseCase", () => { test("writes the default-locale fields on the object, on top of the translations", async () => { const { useCase, metadata } = buildUseCase(); - await useCase.execute({ inputFile: "in.xlsx", post: true, defaultLocale: "en" }); + await useCase.execute({ inputFile: "in.xlsx", post: true, defaultLocale: "en", bumpVersions: false }); const [objects] = metadata.save.mock.calls[0]; expect(objects).toEqual([ @@ -72,7 +88,12 @@ describe("TranslateMetadataUseCase", () => { ], }); - await useCase.execute({ inputFile: "in.xlsx", post: false, defaultLocale: "en" }); + await useCase.execute({ + inputFile: "in.xlsx", + post: false, + defaultLocale: "en", + bumpVersions: false, + }); expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("dataElements.fromName")); }); @@ -84,7 +105,12 @@ describe("TranslateMetadataUseCase", () => { ], }); - await useCase.execute({ inputFile: "in.xlsx", post: false, defaultLocale: "en" }); + await useCase.execute({ + inputFile: "in.xlsx", + post: false, + defaultLocale: "en", + bumpVersions: false, + }); expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("unique field dataElements.name")); }); @@ -100,12 +126,106 @@ describe("TranslateMetadataUseCase", () => { ], }); - await useCase.execute({ inputFile: "in.xlsx", post: false, defaultLocale: undefined }); + await useCase.execute({ + inputFile: "in.xlsx", + post: false, + defaultLocale: undefined, + bumpVersions: false, + }); expect(log.warn).not.toHaveBeenCalled(); }); + + describe("--bump-versions", () => { + test("does not look up data sets/programs when the flag is off", async () => { + const { useCase, dataSets, programs } = buildUseCase(); + + await execute(useCase, { post: true, bumpVersions: false }); + + expect(dataSets.getAll).not.toHaveBeenCalled(); + expect(programs.get).not.toHaveBeenCalled(); + }); + + test("bumps only the data sets/programs using a changed data element", async () => { + const { useCase, dataSets, programs } = buildUseCase(); + + await execute(useCase, { post: true, bumpVersions: true }); + + const [dataSetsPayload] = dataSets.post.mock.calls[0]; + expect(dataSetsPayload.dataSets.map((d: DataSet) => [d.id, d.version])).toEqual([["DS1", 4]]); + + const [programsPosted] = programs.save.mock.calls[0]; + // PR1 has no version yet, so it starts at 1. + expect(programsPosted.map((p: Program) => [p.id, p.version])).toEqual([["PR1", 1]]); + }); + + test("writes nothing in a dry run", async () => { + const { useCase, dataSets, programs } = buildUseCase(); + + await execute(useCase, { post: false, bumpVersions: true }); + + expect(dataSets.post).not.toHaveBeenCalled(); + expect(programs.save).not.toHaveBeenCalled(); + }); + + test("makes no request when no object changed", async () => { + const { useCase, dataSets, programs } = buildUseCase({ fieldTranslations: [] }); + + await execute(useCase, { post: true, bumpVersions: true }); + + expect(dataSets.getAll).not.toHaveBeenCalled(); + expect(programs.get).not.toHaveBeenCalled(); + }); + }); }); +function execute( + useCase: TranslateMetadataUseCase, + options: { post: boolean; bumpVersions: boolean } +): Promise { + return useCase.execute({ inputFile: "in.xlsx", defaultLocale: "en", ...options }); +} + +function buildDataSet(id: string, dataElementId: string, version: Maybe): DataSet { + return { + id: id, + name: `Data set ${id}`, + code: id, + version: version, + skipOffline: false, + categoryCombo: categoryCombo, + dataSetElements: [{ dataElement: { id: dataElementId, name: "Element", code: "DE", categoryCombo } }], + dataInputPeriods: [], + organisationUnits: [], + }; +} + +function buildProgram(id: string, dataElementId: string, version: Maybe): Program { + return { + id: id, + name: `Program ${id}`, + programType: "WITH_REGISTRATION", + version: version, + programStages: [ + { + id: `${id}-stage`, + programStageDataElements: [ + { + dataElement: { + id: dataElementId, + name: "Element", + code: "DE", + valueType: "TEXT", + optionSet: undefined, + }, + displayInReports: false, + }, + ], + }, + ], + }; +} + function buildUseCase(options: { fieldTranslations?: FieldTranslation[] } = {}) { const metadata = { getAllWithTranslations: vi.fn().mockResolvedValue([object]), @@ -113,6 +233,16 @@ function buildUseCase(options: { fieldTranslations?: FieldTranslation[] } = {}) save: vi.fn().mockResolvedValue({ payload: {}, stats: {} }), }; + const dataSets = { + getAll: vi.fn().mockResolvedValue(allDataSets), + post: vi.fn().mockResolvedValue("OK"), + }; + + const programs = { + get: vi.fn().mockResolvedValue(allPrograms), + save: vi.fn().mockResolvedValue(undefined), + }; + const localesRepo: LocalesRepository = { get: vi.fn().mockResolvedValue(locales) }; const schemas: SchemasRepository = { @@ -130,7 +260,9 @@ function buildUseCase(options: { fieldTranslations?: FieldTranslation[] } = {}) locales: localesRepo, schemas: schemas, importTranslations: importTranslations as unknown as ImportTranslationsRepository, + dataSets: dataSets as unknown as DataSetsRepository, + programs: programs as unknown as ProgramsRepository, }); - return { useCase, metadata, localesRepo, schemas, importTranslations }; + return { useCase, metadata, localesRepo, schemas, importTranslations, dataSets, programs }; } diff --git a/src/scripts/commands/translations.ts b/src/scripts/commands/translations.ts index 628eebcd..14b23523 100644 --- a/src/scripts/commands/translations.ts +++ b/src/scripts/commands/translations.ts @@ -9,6 +9,8 @@ import { ImportTranslationsRepositorySpreadsheetRepository } from "data/ImportTr import { ExportTranslationsSpreadsheetRepository } from "data/ExportTranslationsSpreadsheetRepository"; import { MetadataD2Repository } from "data/MetadataD2Repository"; import { SchemasD2Repository } from "data/SchemasD2Repository"; +import { DataSetsD2Repository } from "data/DataSetsD2Repository"; +import { ProgramsD2Repository } from "data/ProgramsD2Repository"; export function getCommand() { const translateFromSpreadsheetCmd = command({ @@ -33,6 +35,12 @@ export function getCommand() { "itself, on top of its translation. Matched by language, so 'en' also matches " + "a locale en_GB. Example: en", }), + bumpVersions: flag({ + long: "bump-versions", + description: + "Increment the version of the data sets/programs using the updated data " + + "elements, so the Capture apps refresh their cached metadata", + }), inputFile: positional({ type: string, displayName: "INPUT_XLSX_PATH", @@ -47,6 +55,8 @@ export function getCommand() { locales: new LocalesD2Repository(api), schemas: new SchemasD2Repository(api), importTranslations: new ImportTranslationsRepositorySpreadsheetRepository(), + dataSets: new DataSetsD2Repository(api), + programs: new ProgramsD2Repository(api), }; await new TranslateMetadataUseCase(repositories).execute(args); From 55c50c7d7a123b22432ad23dceacaa77a7784a5e Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 11:10:49 +0200 Subject: [PATCH 07/10] docs: document --bump-versions in 'translations from-spreadsheet' Co-Authored-By: Claude Opus 5 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fe049399..83b303d2 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ $ yarn start translations from-spreadsheet \ --url='http://USER:PASSWORD@HOST:PORT' \ --save-payload=payload.json \ --default-locale=en \ + --bump-versions \ --post \ translations.xlsx ``` @@ -274,6 +275,7 @@ Notes: - `--default-locale`: locale code of the default (DB) language, `en` for example. Its columns update the object field itself (`formName: English` writes `formName`) on top of creating the translation, so both stay in sync. The match uses only the language part, so `en` also matches a locale `en_GB`. - A warning is shown for columns whose field the instance does not consider translatable (DHIS2 ignores unknown properties, so those columns would silently do nothing), and when `--default-locale` writes a unique field such as `name` or `shortName`, since duplicated values make the whole import fail. +- `--bump-versions`: increment the `version` of every data set and program using an updated data element. Apps typically cache the metadata, so without this the new translations keep showing as the old ones. ### To spreadsheet From 9067e19b603a8f6a16d4c6f16f11a5082670fcc4 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 11:10:49 +0200 Subject: [PATCH 08/10] docs(openspec): spec the --bump-versions cache refresh Co-Authored-By: Claude Opus 5 --- .../spec.md | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/openspec/specs/import-translations-from-spreadsheet/spec.md b/openspec/specs/import-translations-from-spreadsheet/spec.md index 68190e64..39703f6c 100644 --- a/openspec/specs/import-translations-from-spreadsheet/spec.md +++ b/openspec/specs/import-translations-from-spreadsheet/spec.md @@ -88,12 +88,39 @@ Both cases are warnings, not errors: the operator reviews them in the dry run be `fromName`) as translatable for that model - **THEN** a warning naming the model and field is logged once, and the run continues +### Requirement: Refresh the Capture apps cache with --bump-versions + +The Capture apps cache the metadata of each data set and program, and refresh it only when that +object's `version` changes, so an imported translation stays invisible in the app until the owning +objects are bumped. + +The command SHALL provide a `--bump-versions` option that increments the `version` of every data +set and program referencing a data element whose translations changed. The scope is deliberately +data elements only: translating an option set, a tracked entity attribute or a program stage does +not bump anything, even though those are cached too. + +The bump SHALL be opt-in, because it writes objects the operator never listed in the spreadsheet. +It SHALL be skipped entirely, with no lookup request, when no object changed. + +#### Scenario: Only the owners of a changed data element are bumped + +- **WHEN** a data element's translations changed and the command runs with `--bump-versions --post` +- **THEN** the data sets and programs referencing that data element are saved with their version + incremented by one (starting at 1 when unset), and the ones referencing only other data elements + are left untouched + +#### Scenario: Dry run reports the bumps without writing + +- **WHEN** the command runs with `--bump-versions` but without `--post` +- **THEN** the intended bumps are logged and no data set or program is written + ### Requirement: Dry-run by default, apply with --post The command SHALL only persist changes when `--post` is given; otherwise it SHALL validate the payload against the instance (`importMode=VALIDATE`) and report what would change. The `--save-payload` option SHALL write the computed metadata payload to a JSON file in either mode, -so it can be reviewed or replayed. +so it can be reviewed or replayed. The version bumps of `--bump-versions` are posted separately and +are therefore not part of that payload. #### Scenario: Dry run reports without writing From 3e66ac8fabfea0d606cb6d347b9197a03f495574 Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Mon, 3 Aug 2026 11:10:56 +0200 Subject: [PATCH 09/10] style: apply prettier formatting Co-Authored-By: Claude Opus 5 --- README.md | 10 ++++++---- .../ExportTranslationsSpreadsheetRepository.ts | 16 ++++++++++------ .../__tests__/ExportTranslationsUseCase.spec.ts | 5 +---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 83b303d2..f42e72c5 100644 --- a/README.md +++ b/README.md @@ -935,14 +935,16 @@ A config file with the access info of the server and the message webhook details ``` This reports stores data into the `d2-tools.user-roles-authorities-monitoring` datastore. -If some change is detected for a UserRole Authorities a message is generated with three categories: -- New user roles detected with its authorities -- Deleted user roles detected with its authorities -- Updated user roles detected with a list of added and removed authorities +If some change is detected for a UserRole Authorities a message is generated with three categories: + +- New user roles detected with its authorities +- Deleted user roles detected with its authorities +- Updated user roles detected with a list of added and removed authorities If a authority assigned to a UserRole is deprecated (legacy authority, missing or removed app authority, etc) its name will be set to "DEPRECATED_AUTHORITY". Example of the message: + ``` New user roles detected: - NEW USER ROLE (Id: XXXXXXXXXXX) with authorities: diff --git a/src/data/ExportTranslationsSpreadsheetRepository.ts b/src/data/ExportTranslationsSpreadsheetRepository.ts index a9571edd..e8b8f7a4 100644 --- a/src/data/ExportTranslationsSpreadsheetRepository.ts +++ b/src/data/ExportTranslationsSpreadsheetRepository.ts @@ -77,7 +77,9 @@ export class ExportTranslationsSpreadsheetRepository implements ExportTranslatio _.forEach(files, (content, name) => { if (!/^xl\/worksheets\/sheet\d+\.xml$/.test(name)) return; - const xml = decoder.decode(content).replace(/]*?)\/>/, `${pane}`); + const xml = decoder + .decode(content) + .replace(/]*?)\/>/, `${pane}`); files[name] = encoder.encode(xml); }); @@ -150,11 +152,13 @@ export class ExportTranslationsSpreadsheetRepository implements ExportTranslatio const palette = fieldPalette[index % fieldPalette.length]; if (!palette) return []; const base: ColumnStyle = { kind: "base", headerColor: palette.header, bodyColor: palette.base }; - const locales = sheet.locales.map((): ColumnStyle => ({ - kind: "locale", - headerColor: palette.header, - bodyColor: palette.locale, - })); + const locales = sheet.locales.map( + (): ColumnStyle => ({ + kind: "locale", + headerColor: palette.header, + bodyColor: palette.locale, + }) + ); return [base, ...locales]; }); diff --git a/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts b/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts index 0531a1b9..e487d441 100644 --- a/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts +++ b/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts @@ -61,10 +61,7 @@ describe("ExportTranslationsUseCase", () => { const options = exportTranslations.save.mock.calls[0][0]; expect(options.outputFile).toBe("translations.xlsx"); expect(options.includeData).toBe(true); - expect(options.sheets.map((s: { model: string }) => s.model)).toEqual([ - "dataElements", - "indicators", - ]); + expect(options.sheets.map((s: { model: string }) => s.model)).toEqual(["dataElements", "indicators"]); expect(options.sheets[0].fields).toEqual(["name", "formName"]); }); }); From 29ca48bd85d9146db2c69e7b07be7688d26435bd Mon Sep 17 00:00:00 2001 From: Arnau Sanchez Date: Thu, 17 Sep 2026 09:51:54 +0200 Subject: [PATCH 10/10] feat: bare field columns update the field in 'translations from-spreadsheet' A column named as a translatable field with no locale (name, shortName, formName...) now writes the object field itself, matching the layout that 'to-spreadsheet' generates. The name column is written only when the row also has an id/code, otherwise it stays a lookup key. Columns that are not a translatable field of the model are ignored with a warning, and the run logs a per-model summary of the fields updated. Co-Authored-By: Claude Opus 5 --- README.md | 2 +- .../spec.md | 28 +++++++- ...slationsRepositorySpreadsheetRepository.ts | 41 ++++++++++- ...onsRepositorySpreadsheetRepository.spec.ts | 68 ++++++++++++++++++- src/domain/entities/FieldTranslations.ts | 4 +- .../ImportTranslationsRepository.ts | 5 ++ .../usecases/TranslateMetadataUseCase.ts | 39 ++++++++--- .../TranslateMetadataUseCase.spec.ts | 23 ++++++- 8 files changed, 190 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f42e72c5..9eb1ec4b 100644 --- a/README.md +++ b/README.md @@ -270,11 +270,11 @@ Expected format of `xlsx` file: - A Column named `type`/`kind` specifies the DHIS2 entity type (singular). Example: `dataElement`, `dataSet`. - Columns named `id`/`name`/`code` will be used to match the existing object in the database. No need to specify all of them. - Translation columns should have the format: `field:localeName`. A DHIS2 Locale with that name should exist in the database. Example: `formName:French`. +- Field columns (a translatable field with no locale, e.g. `formName`) update the object field itself (`name` is written only when the row also has an `id`/`code`; otherwise it is just the lookup key). Notes: - `--default-locale`: locale code of the default (DB) language, `en` for example. Its columns update the object field itself (`formName: English` writes `formName`) on top of creating the translation, so both stay in sync. The match uses only the language part, so `en` also matches a locale `en_GB`. -- A warning is shown for columns whose field the instance does not consider translatable (DHIS2 ignores unknown properties, so those columns would silently do nothing), and when `--default-locale` writes a unique field such as `name` or `shortName`, since duplicated values make the whole import fail. - `--bump-versions`: increment the `version` of every data set and program using an updated data element. Apps typically cache the metadata, so without this the new translations keep showing as the old ones. ### To spreadsheet diff --git a/openspec/specs/import-translations-from-spreadsheet/spec.md b/openspec/specs/import-translations-from-spreadsheet/spec.md index 39703f6c..acf81db0 100644 --- a/openspec/specs/import-translations-from-spreadsheet/spec.md +++ b/openspec/specs/import-translations-from-spreadsheet/spec.md @@ -52,6 +52,30 @@ the current one SHALL be posted. - **WHEN** an object has a French `FORM_NAME` translation and the sheet only carries a Spanish one - **THEN** the posted object keeps the French translation and gains the Spanish one +### Requirement: Bare field columns update the object field + +A column headed exactly as a translatable field of the row's model, with no locale (`formName`, +`shortName`), SHALL write that field on the object. This is the layout `to-spreadsheet` generates +(source column followed by its locale columns), so an unedited export round-trips as a no-op and +edited source cells are posted without any option. Columns matching no translatable field of the +model SHALL be ignored with a warning once per sheet. Empty cells SHALL not blank the field. + +`name` doubles as a lookup key: it SHALL be written only when the row also carries an `id` or +`code`, otherwise a case-insensitive name match would rename the object to the sheet's spelling. +The run SHALL log, per model, how many rows update which fields, so field changes are visible in +the dry run alongside the translations. + +#### Scenario: Edited source column is posted + +- **WHEN** a row has `id`, `name` and `name: French` columns and the `name` cell differs from the + object's current name +- **THEN** the posted object has the new `name` and the French `NAME` translation + +#### Scenario: Name-only rows are not renamed + +- **WHEN** a row identifies the object only by `name` +- **THEN** the name is used to find the object and is not written back as a field + ### Requirement: Default-locale columns also update the object field The command SHALL provide a `--default-locale` option taking the locale code of the instance's @@ -77,8 +101,8 @@ declares in `/api/schemas`, and SHALL warn once per model/field (not per row) wh not translatable there — DHIS2 ignores unknown properties, so such a column would post successfully while changing nothing. -It SHALL also warn when `--default-locale` writes a unique-constrained field (`name`, -`shortName`), because a duplicated value makes the whole metadata payload fail to validate. +It SHALL also warn when a column writes a unique-constrained field (`name`, `shortName`), +because a duplicated value makes the whole metadata payload fail to validate. Both cases are warnings, not errors: the operator reviews them in the dry run before posting. diff --git a/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts b/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts index 746417cb..1d31b943 100644 --- a/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts +++ b/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts @@ -25,7 +25,8 @@ const columnsMapping = { - id/uid - code - name - - FIELD1: LANGUAGE_NAME1 + - FIELD1 -> updates the object field itself (example: formName) + - FIELD1: LANGUAGE_NAME1 -> updates the translation - FIELD2: LANGUAGE_NAME2 - ... @@ -118,9 +119,41 @@ export class ImportTranslationsRepositorySpreadsheetRepository implements Import .value(); const translations = entries.map(entry => entry.translation); - const fieldValues: FieldValues = Object.assign({}, ...entries.map(entry => entry.fieldValue)); + const pluralModel = getPluralModel(model); + const bareFieldValues = this.getBareFieldValues(row, pluralModel, identifier, options); - return { model: getPluralModel(model), identifier: identifier, translations, fields: fieldValues }; + // An explicit default-locale column wins over the bare column of the same field. + const fieldValues: FieldValues = Object.assign( + {}, + bareFieldValues, + ...entries.map(entry => entry.fieldValue) + ); + + return { model: pluralModel, identifier: identifier, translations, fields: fieldValues }; + } + + /* A bare column (no locale) named exactly as a translatable field of the model writes that + field. The name column doubles as the lookup key: it's written only when the row also has + an id or code, otherwise a case-insensitive lookup match would rename the object. */ + private getBareFieldValues( + row: Row, + model: string, + identifier: Identifier, + options: GetFieldTranslationsOptions + ): FieldValues { + const translatableFields = options.translatableFields[model] ?? []; + const identifierColumns = _.flatMap(columnsMapping); + const hasIdOrCode = Boolean(identifier.id || identifier.code); + + return _(row) + .toPairs() + .reject(([column]) => column.includes(":")) + .reject(([column]) => + column === "name" ? !hasIdOrCode : identifierColumns.includes(column.toLowerCase()) + ) + .filter(([column, text]) => (translatableFields.includes(column) ? Boolean(text) : false)) + .fromPairs() + .value(); } private getHeaderValue(row: Row, column: ColumnsMappingKey): Maybe { @@ -135,6 +168,8 @@ export class ImportTranslationsRepositorySpreadsheetRepository implements Import type Row = Record; +type Identifier = FieldTranslation["identifier"]; + type ColumnsMappingKey = keyof typeof columnsMapping; interface ColumnEntry { diff --git a/src/data/__tests__/ImportTranslationsRepositorySpreadsheetRepository.spec.ts b/src/data/__tests__/ImportTranslationsRepositorySpreadsheetRepository.spec.ts index 4f33ea3b..e235de14 100644 --- a/src/data/__tests__/ImportTranslationsRepositorySpreadsheetRepository.spec.ts +++ b/src/data/__tests__/ImportTranslationsRepositorySpreadsheetRepository.spec.ts @@ -2,7 +2,8 @@ import fs from "fs"; import os from "os"; import path from "path"; import * as XLSX from "xlsx"; -import { afterAll, describe, expect, test } from "vitest"; +import { afterAll, beforeEach, describe, expect, test, vi } from "vitest"; +import log from "utils/log"; import { ImportTranslationsRepositorySpreadsheetRepository } from "../ImportTranslationsRepositorySpreadsheetRepository"; import { FieldTranslation } from "domain/entities/FieldTranslations"; import { Locale } from "domain/entities/Locale"; @@ -13,10 +14,19 @@ const locales: Locale[] = [ { id: "2", name: "Spanish (Spain)", locale: "es" }, ]; +const translatableFields = { + dataElements: ["name", "shortName", "formName"], + validationRules: ["leftSideDescription"], +}; + const header = ["type", "id", "name: English", "formName: English", "formName: Spanish"]; const row = ["dataElement", "abc", "Malaria cases", "Malaria form", "Formulario malaria"]; describe("ImportTranslationsRepositorySpreadsheetRepository", () => { + beforeEach(() => { + vi.spyOn(log, "warn").mockImplementation(() => undefined); + }); + test("without a default locale, columns only produce translations", async () => { const fieldTranslation = await getFirst(undefined); @@ -46,6 +56,60 @@ describe("ImportTranslationsRepositorySpreadsheetRepository", () => { expect(fieldTranslation?.fields).toEqual({ formName: "Formulario malaria" }); }); + describe("bare field columns (no locale)", () => { + test("write the field, not a translation", async () => { + const fieldTranslation = await getFirst(undefined, { + header: ["Type", "UID", "Name", "name", "shortName", "formName: Spanish"], + row: ["dataElement", "abc", "Old name", "Malaria cases", "Malaria", "Formulario malaria"], + }); + + expect(fieldTranslation?.identifier).toEqual({ + id: "abc", + name: "Malaria cases", + code: undefined, + }); + expect(fieldTranslation?.fields).toEqual({ name: "Malaria cases", shortName: "Malaria" }); + expect(fieldTranslation?.translations).toEqual([ + { property: "FORM_NAME", locale: "es", value: "Formulario malaria" }, + ]); + }); + + test("name is only a lookup key when the row has no id/code", async () => { + const fieldTranslation = await getFirst(undefined, { + header: ["type", "name", "shortName"], + row: ["dataElement", "Malaria cases", "Malaria"], + }); + + expect(fieldTranslation?.identifier).toEqual({ + id: undefined, + name: "Malaria cases", + code: undefined, + }); + expect(fieldTranslation?.fields).toEqual({ shortName: "Malaria" }); + }); + + test("ignore columns that are not a translatable field of the model, with a warning", async () => { + const fieldTranslation = await getFirst(undefined, { + header: ["type", "id", "shortName", "Comments"], + row: ["dataElement", "abc", "Malaria", "Reviewed"], + }); + + expect(fieldTranslation?.fields).toEqual({ shortName: "Malaria" }); + expect(log.warn).toHaveBeenCalledWith( + expect.stringContaining("not a translatable field of dataElements: Comments") + ); + }); + + test("an explicit default-locale column wins over the bare column", async () => { + const fieldTranslation = await getFirst("en", { + header: ["type", "id", "shortName", "shortName: English"], + row: ["dataElement", "abc", "Malaria", "Malaria (en)"], + }); + + expect(fieldTranslation?.fields).toEqual({ shortName: "Malaria (en)" }); + }); + }); + test("spaced column names are converted to the object field name", async () => { const fieldTranslation = await getFirst("en", { header: ["type", "id", "Left side description: English"], @@ -71,7 +135,7 @@ async function getFirst( ): Promise> { const inputFile = writeSpreadsheet([rows.header, rows.row]); const repository = new ImportTranslationsRepositorySpreadsheetRepository(); - const fieldTranslations = await repository.get({ inputFile, locales, defaultLocale }); + const fieldTranslations = await repository.get({ inputFile, locales, defaultLocale, translatableFields }); return fieldTranslations[0]; } diff --git a/src/domain/entities/FieldTranslations.ts b/src/domain/entities/FieldTranslations.ts index ffd673dc..8ff5d086 100644 --- a/src/domain/entities/FieldTranslations.ts +++ b/src/domain/entities/FieldTranslations.ts @@ -6,8 +6,8 @@ export interface FieldTranslation { model: string; // plural identifier: Partial<{ id: string; name: string; code: string }>; translations: Translation[]; - /* Values to write on the object itself, not as a translation. Filled from the columns whose - locale is the default locale (see option --default-locale). */ + /* Values to write on the object itself, not as a translation. Filled from the bare field + columns (no locale) and from the columns of the default locale (option --default-locale). */ fields: FieldValues; } diff --git a/src/domain/repositories/ImportTranslationsRepository.ts b/src/domain/repositories/ImportTranslationsRepository.ts index d2205245..b9cb1cfb 100644 --- a/src/domain/repositories/ImportTranslationsRepository.ts +++ b/src/domain/repositories/ImportTranslationsRepository.ts @@ -1,6 +1,8 @@ import { Async } from "domain/entities/Async"; import { FieldTranslations } from "domain/entities/FieldTranslations"; import { Locale, LocaleCode } from "domain/entities/Locale"; +import { MetadataModel } from "domain/entities/MetadataObject"; +import { TranslatableField } from "domain/entities/Translation"; export interface ImportTranslationsRepository { get(options: GetFieldTranslationsOptions): Async; @@ -12,4 +14,7 @@ export interface GetFieldTranslationsOptions { /* Columns of this locale are also written to the object field itself, not only as a translation. Matched by language, so "en" also matches a column mapped to "en_GB". */ defaultLocale?: LocaleCode; + /* Translatable fields of each model (plural). A bare column (no locale) named as one of them + writes the field itself. */ + translatableFields: Record; } diff --git a/src/domain/usecases/TranslateMetadataUseCase.ts b/src/domain/usecases/TranslateMetadataUseCase.ts index 5321410e..e483e7e5 100644 --- a/src/domain/usecases/TranslateMetadataUseCase.ts +++ b/src/domain/usecases/TranslateMetadataUseCase.ts @@ -10,7 +10,7 @@ import { LocalesRepository } from "domain/repositories/LocalesRepository"; import { SchemasRepository } from "domain/repositories/SchemasRepository"; import { DataSetsRepository } from "domain/repositories/DataSetsRepository"; import { ProgramsRepository } from "domain/repositories/ProgramsRepository"; -import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; +import { MetadataModel, MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; import { Maybe } from "utils/ts-utils"; import { getId, Id } from "domain/entities/Base"; import { LocaleCode } from "domain/entities/Locale"; @@ -98,15 +98,20 @@ export class TranslateMetadataUseCase { } private async getObjectsToPost(options: Options) { - const locales = await this.repositories.locales.get(); + const [locales, translatableFieldsByModel] = await Promise.all([ + this.repositories.locales.get(), + this.repositories.schemas.getTranslatableFields(), + ]); const fieldTranslations = await this.repositories.importTranslations.get({ inputFile: options.inputFile, locales: locales, defaultLocale: options.defaultLocale, + translatableFields: translatableFieldsByModel, }); - await this.validateFields(fieldTranslations); + this.validateFields(fieldTranslations, translatableFieldsByModel); + this.logFieldUpdates(fieldTranslations); const models = _(fieldTranslations) .map(o => o.model) @@ -122,11 +127,12 @@ export class TranslateMetadataUseCase { } /* Warn about columns whose field the instance does not consider translatable (DHIS2 ignores - unknown properties, so those columns would silently do nothing) and about default-locale - columns writing unique-constrained fields. One warning per model/field, not per row. */ - private async validateFields(fieldTranslations: FieldTranslations): Async { - const translatableFieldsByModel = await this.repositories.schemas.getTranslatableFields(); - + unknown properties, so those columns would silently do nothing) and about columns writing + unique-constrained fields. One warning per model/field, not per row. */ + private validateFields( + fieldTranslations: FieldTranslations, + translatableFieldsByModel: Record + ): void { const usages = _(fieldTranslations) .flatMap(({ model, fields, translations }) => { const fromFields = _.keys(fields).map(field => ({ model, field, isFieldValue: true })); @@ -149,13 +155,28 @@ export class TranslateMetadataUseCase { log.warn(`Field not translatable, column ignored: ${model}.${field}`); } else if (isFieldValue && uniqueTranslatableFields.includes(field)) { log.warn( - `Default locale writes the unique field ${model}.${field}: ` + + `Writing the unique field ${model}.${field}: ` + `duplicated values will make the import fail` ); } }); } + /* Field updates go beyond translations, make them visible in the (dry) run output. */ + private logFieldUpdates(fieldTranslations: FieldTranslations): void { + _(fieldTranslations) + .filter(({ fields }) => !_.isEmpty(fields)) + .groupBy(({ model }) => model) + .forEach((group, model) => { + const fields = _(group) + .flatMap(({ fields }) => _.keys(fields)) + .uniq() + .sort() + .join(", "); + log.info(`${model}: ${group.length} rows update fields [${fields}]`); + }); + } + private addTranslations( objects: MetadataObjectWithTranslations[], fieldTranslations: FieldTranslations diff --git a/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts b/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts index d686a181..17044be0 100644 --- a/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts +++ b/src/domain/usecases/__tests__/TranslateMetadataUseCase.spec.ts @@ -63,6 +63,27 @@ describe("TranslateMetadataUseCase", () => { ); }); + test("passes the translatable fields of the instance to the spreadsheet reader", async () => { + const { useCase, importTranslations } = buildUseCase(); + + await execute(useCase, { post: false, bumpVersions: false }); + + expect(importTranslations.get).toHaveBeenCalledWith( + expect.objectContaining({ + translatableFields: { dataElements: expect.arrayContaining(["name"]) }, + }) + ); + }); + + test("logs a summary of the fields updated per model", async () => { + const { useCase } = buildUseCase(); + vi.spyOn(log, "info").mockImplementation(() => undefined); + + await execute(useCase, { post: false, bumpVersions: false }); + + expect(log.info).toHaveBeenCalledWith("dataElements: 1 rows update fields [formName]"); + }); + test("writes the default-locale fields on the object, on top of the translations", async () => { const { useCase, metadata } = buildUseCase(); @@ -98,7 +119,7 @@ describe("TranslateMetadataUseCase", () => { expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("dataElements.fromName")); }); - test("warns when the default locale writes a unique field", async () => { + test("warns when a column writes a unique field", async () => { const { useCase } = buildUseCase({ fieldTranslations: [ { ...fieldTranslation, translations: [], fields: { name: "Malaria cases (new)" } },