diff --git a/README.md b/README.md index d7a1a0dc..1f915109 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,8 @@ Notes: ## Translations +### From spreadsheet + Update objects from spreadsheet. Update any type of DHIS2 metadata object using a xlsx spreadsheet as a data source: ```shell @@ -267,6 +269,28 @@ Expected format of `xlsx` file: - Columns named `id`/`name`/`code` will be used to match the existing object in the database. No need to specify all of them. - Translation columns should have the format: `field:localeName`. A DHIS2 Locale with that name should exist in the database. Example: `formName:French`. +### To spreadsheet + +Generate a translations spreadsheet from the metadata objects of a DHIS2 instance. The output is re-importable by `from-spreadsheet`: + +```shell +$ yarn start translations to-spreadsheet \ + --url='http://USER:PASSWORD@HOST:PORT' \ + --models='dataElements[name,formName],indicators[name]' \ + --locales='Spanish,French' \ + --include-data \ + translations.xlsx +``` + +Notes: + +- `--models`: comma-separated list of models to export. Each model must specify its translatable fields with `[field1,field2]` (e.g. `indicators[name,shortName]`); a model without fields raises an error. +- `--locales`: comma-separated list of locale names to include as columns, in the order given. The match ignores any ` (...)` suffix, so `Spanish` matches a `Spanish (Spain)` locale. +- `--include-data`: write one row per object with the source values and the existing translations. When omitted, only the header row is written (a column template). +- One sheet (tab) is generated per model type. +- Columns: `Type`, `UID`, then a group per field: the base source column `` followed by one `: ` column per selected locale. +- Each field group is color-coded (bold colored header, source column highlighted, translation cells lightly tinted) so the grid is easy to scan. The header row and the first three columns (`Type`, `UID` and the first source column) are frozen, and columns within a field group share the same width. + ## Events ### Detect events assigned to organisation units outside their enrollment diff --git a/package.json b/package.json index 54ef1287..e535bdae 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "csv-reader": "^1.0.8", "csv-writer": "^1.6.0", "d2-utilizr": "^0.2.16", + "fflate": "^0.8.3", "file-system-cache": "^2.0.0", "json-diff": "^0.7.3", "jsonfile": "^6.1.0", @@ -41,7 +42,8 @@ "random-seed": "^0.3.0", "simple-node-logger": "^21.8.12", "socks-proxy-agent": "^8.0.3", - "xlsx": "^0.18.5" + "xlsx": "^0.18.5", + "xlsx-js-style": "^1.2.0" }, "devDependencies": { "@types/har-format": "^1.2.13", diff --git a/src/data/ExportTranslationsSpreadsheetRepository.ts b/src/data/ExportTranslationsSpreadsheetRepository.ts new file mode 100644 index 00000000..a9571edd --- /dev/null +++ b/src/data/ExportTranslationsSpreadsheetRepository.ts @@ -0,0 +1,246 @@ +import _ from "lodash"; +import fs from "fs"; +import XLSX from "xlsx-js-style"; +import { unzipSync, zipSync } from "fflate"; +import { Async } from "domain/entities/Async"; +import { Locale } from "domain/entities/Locale"; +import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; +import { ModelTranslationsExport } from "domain/entities/ModelTranslationsExport"; +import { translationFieldToProperty } from "domain/entities/Translation"; +import { + ExportTranslationsOptions, + ExportTranslationsRepository, +} from "domain/repositories/ExportTranslationsRepository"; +import { getSingularModel } from "./dhis2-utils"; +import log from "utils/log"; + +/* Generates a xlsx (one sheet per model) re-importable by `from-spreadsheet`. + + Columns: Type, UID, then a group per field: the base source column followed + by one : translation column per locale. + + When includeData is false, only the header row is written (a column template). When true, + one row per object is written with the source values and the existing translations. + + Each field group is colored with its own hue (stronger on the header and on the base/source + column, a light tint on the translation cells) so the wide grid stays easy to scan. +*/ +export class ExportTranslationsSpreadsheetRepository implements ExportTranslationsRepository { + async save(options: ExportTranslationsOptions): Async { + const { outputFile, sheets, includeData } = options; + const workbook = XLSX.utils.book_new(); + + sheets.forEach(sheet => { + const { name, header, rows } = this.buildSheet(sheet, includeData); + const worksheet = XLSX.utils.aoa_to_sheet([header, ...rows]); + + worksheet["!cols"] = this.getColumnWidths(sheet, header, rows); + worksheet["!autofilter"] = { + ref: XLSX.utils.encode_range({ + s: { r: 0, c: 0 }, + e: { r: rows.length, c: header.length - 1 }, + }), + }; + this.applyStyles(worksheet, this.getColumnStyles(sheet)); + + XLSX.utils.book_append_sheet(workbook, worksheet, name); + }); + + log.info(`Save file ${outputFile}`); + XLSX.writeFile(workbook, outputFile); + + // xlsx-js-style cannot write freeze panes, so patch the file afterwards. + this.freezePanes(outputFile, { rows: 1, columns: 3 }); + } + + /* Pure transformation of a model export into the sheet name, header row and data rows. + When includeData is false, no data rows are produced (a header-only template). */ + buildSheet(sheet: ModelTranslationsExport, includeData: boolean): SheetData { + const header = this.getHeader(sheet); + const rows = includeData ? sheet.objects.map(object => this.getRow(object, sheet)) : []; + const name = sheet.model.replace(/[^a-zA-Z0-9-_()\s]/g, "-").slice(0, 31); + return { name, header, rows }; + } + + /* Freeze the first `rows` rows and `columns` columns on every sheet by injecting a + element into the worksheet XML (not supported by the writer itself). */ + private freezePanes(outputFile: string, options: { rows: number; columns: number }): void { + const { rows, columns } = options; + const files = unzipSync(fs.readFileSync(outputFile)); + const decoder = new TextDecoder(); + const encoder = new TextEncoder(); + + const topLeftCell = XLSX.utils.encode_cell({ r: rows, c: columns }); + const pane = + ``; + + _.forEach(files, (content, name) => { + if (!/^xl\/worksheets\/sheet\d+\.xml$/.test(name)) return; + const xml = decoder.decode(content).replace(/]*?)\/>/, `${pane}`); + files[name] = encoder.encode(xml); + }); + + fs.writeFileSync(outputFile, Buffer.from(zipSync(files))); + } + + /* Color the header row and tint each column according to its field group. */ + private applyStyles(worksheet: XLSX.WorkSheet, columns: ColumnStyle[]): void { + const range = XLSX.utils.decode_range(worksheet["!ref"] ?? "A1"); + + for (let c = range.s.c; c <= range.e.c; c++) { + const column = columns[c]; + if (!column) continue; + + for (let r = range.s.r; r <= range.e.r; r++) { + const address = XLSX.utils.encode_cell({ r, c }); + const cell = worksheet[address] ?? (worksheet[address] = { t: "s", v: "" }); + cell.s = r === 0 ? headerStyle(column) : bodyStyle(column); + } + } + } + + /* Column widths (in characters), clamped to a readable range. The columns of a field group + (base + its locale columns) share a single width so empty translation columns aren't narrow. */ + private getColumnWidths( + sheet: ModelTranslationsExport, + header: string[], + dataRows: string[][] + ): XLSX.ColInfo[] { + const minWidth = 12; + const maxWidth = 60; + const padding = 2; + + const contentWidth = (index: number) => + _(dataRows) + .map(row => (row[index] ?? "").length) + .push((header[index] ?? "").length) + .max() ?? 0; + + const clamp = (width: number) => _.clamp(width + padding, minWidth, maxWidth); + + const widths = header.map((_column, index) => clamp(contentWidth(index))); + + // Unify the width within each field group: [base, locale1, locale2, ...]. + const groupSize = 1 + sheet.locales.length; + sheet.fields.forEach((_field, fieldIndex) => { + const start = 2 + fieldIndex * groupSize; + const indexes = _.range(start, start + groupSize); + const groupWidth = _(indexes).map(contentWidth).max() ?? 0; + indexes.forEach(index => (widths[index] = clamp(groupWidth))); + }); + + return widths.map(width => ({ wch: width })); + } + + private getHeader(sheet: ModelTranslationsExport): string[] { + const fieldColumns = sheet.fields.flatMap(field => [ + field, + ...sheet.locales.map(locale => this.getTranslationColumn(field, locale)), + ]); + + return ["Type", "UID", ...fieldColumns]; + } + + /* Per-column color descriptors, aligned with getHeader. */ + private getColumnStyles(sheet: ModelTranslationsExport): ColumnStyle[] { + const meta: ColumnStyle = { kind: "meta", headerColor: metaColor.header, bodyColor: metaColor.body }; + + const fieldColumns = sheet.fields.flatMap((_field, index): ColumnStyle[] => { + const palette = fieldPalette[index % fieldPalette.length]; + if (!palette) return []; + const base: ColumnStyle = { kind: "base", headerColor: palette.header, bodyColor: palette.base }; + const locales = sheet.locales.map((): ColumnStyle => ({ + kind: "locale", + headerColor: palette.header, + bodyColor: palette.locale, + })); + return [base, ...locales]; + }); + + return [meta, meta, ...fieldColumns]; + } + + private getRow(object: MetadataObjectWithTranslations, sheet: ModelTranslationsExport): string[] { + const fieldCells = sheet.fields.flatMap(field => [ + getFieldValue(object, field), + ...sheet.locales.map(locale => this.getTranslationValue(object, field, locale)), + ]); + + return [getSingularModel(sheet.model), object.id, ...fieldCells]; + } + + private getTranslationColumn(field: string, locale: Locale): string { + return `${field}: ${locale.name}`; + } + + private getTranslationValue( + object: MetadataObjectWithTranslations, + field: string, + locale: Locale + ): string { + const property = translationFieldToProperty(field); + const translation = object.translations.find( + t => t.property === property && t.locale === locale.locale + ); + return translation?.value ?? ""; + } +} + +function getFieldValue(object: MetadataObjectWithTranslations, field: string): string { + // Objects are fetched with `:owner`, so they carry all owner fields at runtime even + // though the type only declares id/name/code/translations. + const value = (object as unknown as Record)[field]; + return typeof value === "string" ? value : ""; +} + +export interface SheetData { + name: string; + header: string[]; + rows: string[][]; +} + +interface ColumnStyle { + kind: "meta" | "base" | "locale"; + headerColor: string; // fill for the header row + bodyColor: string; // fill for the data rows +} + +// Header/body fills per field group (strong header, stronger base column, light translation cells). +const fieldPalette = [ + { header: "8EAADB", base: "C9D6EC", locale: "E4EBF5" }, // blue + { header: "A9D08E", base: "D2E4C4", locale: "EAF2E1" }, // green + { header: "FFD966", base: "FFE9A8", locale: "FFF4D4" }, // gold + { header: "B89BD9", base: "DAC9EC", locale: "ECE3F5" }, // purple + { header: "F4B183", base: "F9D2B6", locale: "FCE8DB" }, // orange + { header: "7FC5BD", base: "BCE0DB", locale: "DEF0ED" }, // teal +]; + +const metaColor = { header: "BFBFBF", body: "F2F2F2" }; + +function fill(rgb: string) { + return { patternType: "solid", fgColor: { rgb } }; +} + +function border(rgb: string) { + const side = { style: "thin", color: { rgb } }; + return { top: side, bottom: side, left: side, right: side }; +} + +function headerStyle(column: ColumnStyle) { + return { + fill: fill(column.headerColor), + font: { bold: true, color: { rgb: "000000" } }, + alignment: { horizontal: "center", vertical: "center", wrapText: true }, + border: border("808080"), + }; +} + +function bodyStyle(column: ColumnStyle) { + return { + fill: fill(column.bodyColor), + font: { bold: column.kind === "base" }, + alignment: { vertical: "top", wrapText: true }, + border: border("D9D9D9"), + }; +} diff --git a/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts b/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts index da6675f1..7f15b205 100644 --- a/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts +++ b/src/data/ImportTranslationsRepositorySpreadsheetRepository.ts @@ -9,6 +9,7 @@ import { SpreadsheetXlsxDataSource } from "domain/repositories/SpreadsheetXlsxRe import log from "utils/log"; import { Maybe } from "utils/ts-utils"; import { getPluralModel } from "./dhis2-utils"; +import { translationFieldToProperty } from "domain/entities/Translation"; const columnsMapping = { id: ["id", "uid"], @@ -96,7 +97,7 @@ export class ImportTranslationsRepositorySpreadsheetRepository implements Import if (isFirstRow) warn(`Locale not found in DB: name=${localeName}`); return undefined; } else { - const property = _.upperCase(field).replace(/\s+/g, "_"); + const property = translationFieldToProperty(field); return { property: property, locale: locale.locale, value: text }; } }); diff --git a/src/data/__tests__/ExportTranslationsSpreadsheetRepository.spec.ts b/src/data/__tests__/ExportTranslationsSpreadsheetRepository.spec.ts new file mode 100644 index 00000000..948c77ab --- /dev/null +++ b/src/data/__tests__/ExportTranslationsSpreadsheetRepository.spec.ts @@ -0,0 +1,121 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import XLSX from "xlsx-js-style"; +import { afterEach, describe, expect, test } from "vitest"; +import { ExportTranslationsSpreadsheetRepository } from "../ExportTranslationsSpreadsheetRepository"; +import { ModelTranslationsExport } from "domain/entities/ModelTranslationsExport"; +import { Locale } from "domain/entities/Locale"; +import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; + +const french: Locale = { id: "1", name: "French", locale: "fr" }; +const spanish: Locale = { id: "2", name: "Spanish", locale: "es" }; + +const repository = new ExportTranslationsSpreadsheetRepository(); + +describe("ExportTranslationsSpreadsheetRepository.buildSheet", () => { + test("builds the header: Type, UID, then a group per field with locales in order", () => { + const { header } = repository.buildSheet(buildSheetExport(), true); + + expect(header).toEqual([ + "Type", + "UID", + "name", + "name: French", + "name: Spanish", + "formName", + "formName: French", + "formName: Spanish", + ]); + }); + + test("with includeData=false writes no data rows (header-only template)", () => { + const { rows } = repository.buildSheet(buildSheetExport(), false); + expect(rows).toEqual([]); + }); + + test("fills source values + translations, singular Type, blanks for missing field/translation", () => { + const { rows } = repository.buildSheet(buildSheetExport(), true); + + // Type is singular; missing translation (name: Spanish) and missing field value are blank. + expect(rows[0]).toEqual([ + "dataElement", + "id1", + "Hello", + "Bonjour", // NAME / fr + "", // NAME / es (missing) + "HelloForm", // formName source value + "", // FORM_NAME / fr (missing) + "Hola form", // FORM_NAME / es + ]); + + // Second object has no formName field at all -> base column blank. + expect(rows[1]).toEqual(["dataElement", "id2", "Second", "", "", "", "", ""]); + }); +}); + +describe("ExportTranslationsSpreadsheetRepository.save (file round-trip)", () => { + const outputFile = path.join(os.tmpdir(), `export-translations-${Date.now()}.xlsx`); + + afterEach(() => { + if (fs.existsSync(outputFile)) fs.rmSync(outputFile); + }); + + test("writes one sheet per model with a header and autofilter", async () => { + await repository.save({ outputFile, sheets: [buildSheetExport()], includeData: true }); + + const workbook = XLSX.readFile(outputFile); + expect(workbook.SheetNames).toEqual(["dataElements"]); + + const worksheet = getSheet(workbook, "dataElements"); + const rows = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: "" }); + expect(rows[0]?.[0]).toBe("Type"); + expect(rows).toHaveLength(3); // header + 2 objects + expect(worksheet["!autofilter"]?.ref).toBe("A1:H3"); + }); + + test("with includeData=false the sheet has only the header row", async () => { + await repository.save({ outputFile, sheets: [buildSheetExport()], includeData: false }); + + const workbook = XLSX.readFile(outputFile); + const worksheet = getSheet(workbook, "dataElements"); + const rows = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: "" }); + expect(rows).toHaveLength(1); + expect(worksheet["!autofilter"]?.ref).toBe("A1:H1"); + }); +}); + +function getSheet(workbook: XLSX.WorkBook, name: string): XLSX.WorkSheet { + const worksheet = workbook.Sheets[name]; + if (!worksheet) throw new Error(`Sheet not found: ${name}`); + return worksheet; +} + +function buildObject(data: Partial & { formName?: string }) { + return { + model: "dataElements", + code: undefined, + translations: [], + ...data, + } as MetadataObjectWithTranslations; +} + +function buildSheetExport(): ModelTranslationsExport { + return { + model: "dataElements", + fields: ["name", "formName"], + locales: [french, spanish], + objects: [ + buildObject({ + id: "id1", + name: "Hello", + formName: "HelloForm", + translations: [ + { property: "NAME", locale: "fr", value: "Bonjour" }, + { property: "FORM_NAME", locale: "es", value: "Hola form" }, + ], + }), + buildObject({ id: "id2", name: "Second" }), + ], + }; +} diff --git a/src/data/dhis2-utils.ts b/src/data/dhis2-utils.ts index 2d9e5b01..c122883b 100644 --- a/src/data/dhis2-utils.ts +++ b/src/data/dhis2-utils.ts @@ -79,3 +79,7 @@ export function promiseMap(inputValues: T[], mapper: (value: T) => Promise export function getPluralModel(model: string): string { return model.endsWith("s") ? model : model + "s"; } + +export function getSingularModel(model: string): string { + return model.endsWith("s") ? model.slice(0, -1) : model; +} diff --git a/src/domain/entities/ModelTranslationsExport.ts b/src/domain/entities/ModelTranslationsExport.ts new file mode 100644 index 00000000..f1a0f1ce --- /dev/null +++ b/src/domain/entities/ModelTranslationsExport.ts @@ -0,0 +1,9 @@ +import { Locale } from "./Locale"; +import { MetadataObjectWithTranslations } from "./MetadataObject"; + +export interface ModelTranslationsExport { + model: string; // plural, e.g. "dataElements" + fields: string[]; // e.g. ["name", "formName"] + locales: Locale[]; // column locales + objects: MetadataObjectWithTranslations[]; +} diff --git a/src/domain/entities/Translation.ts b/src/domain/entities/Translation.ts index 59815315..46c2eaf6 100644 --- a/src/domain/entities/Translation.ts +++ b/src/domain/entities/Translation.ts @@ -1,3 +1,4 @@ +import _ from "lodash"; import { LocaleCode } from "./Locale"; export interface Translation { @@ -6,6 +7,13 @@ export interface Translation { value: string; } +/* Convert a spreadsheet field name into a DHIS2 translation property. Handles multi-word fields + (more than one underscore) and both camelCase and spaced inputs. + Examples: "formName" -> "FORM_NAME", "leftSideDescription" -> "LEFT_SIDE_DESCRIPTION". */ +export function translationFieldToProperty(field: string): string { + return _.snakeCase(field).toUpperCase(); +} + export interface ModelTranslations { model: string; translations: Translation[]; diff --git a/src/domain/entities/__tests__/Translation.spec.ts b/src/domain/entities/__tests__/Translation.spec.ts new file mode 100644 index 00000000..2085bed6 --- /dev/null +++ b/src/domain/entities/__tests__/Translation.spec.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "vitest"; +import { translationFieldToProperty } from "../Translation"; + +describe("translationFieldToProperty", () => { + test("converts simple camelCase fields", () => { + expect(translationFieldToProperty("name")).toBe("NAME"); + expect(translationFieldToProperty("shortName")).toBe("SHORT_NAME"); + expect(translationFieldToProperty("formName")).toBe("FORM_NAME"); + expect(translationFieldToProperty("description")).toBe("DESCRIPTION"); + expect(translationFieldToProperty("content")).toBe("CONTENT"); + }); + + test("handles multi-word fields (more than one underscore)", () => { + expect(translationFieldToProperty("leftSideDescription")).toBe("LEFT_SIDE_DESCRIPTION"); + expect(translationFieldToProperty("rightSideDescription")).toBe("RIGHT_SIDE_DESCRIPTION"); + expect(translationFieldToProperty("executionDateLabel")).toBe("EXECUTION_DATE_LABEL"); + }); + + test("is tolerant of spaced and already-uppercased inputs", () => { + expect(translationFieldToProperty("Left side description")).toBe("LEFT_SIDE_DESCRIPTION"); + expect(translationFieldToProperty("SHORT_NAME")).toBe("SHORT_NAME"); + }); +}); diff --git a/src/domain/repositories/ExportTranslationsRepository.ts b/src/domain/repositories/ExportTranslationsRepository.ts new file mode 100644 index 00000000..2efe7e89 --- /dev/null +++ b/src/domain/repositories/ExportTranslationsRepository.ts @@ -0,0 +1,12 @@ +import { Async } from "domain/entities/Async"; +import { ModelTranslationsExport } from "domain/entities/ModelTranslationsExport"; + +export interface ExportTranslationsRepository { + save(options: ExportTranslationsOptions): Async; +} + +export interface ExportTranslationsOptions { + outputFile: string; + sheets: ModelTranslationsExport[]; // one per model + includeData: boolean; // false => header row only; true => full rows (source + translations) +} diff --git a/src/domain/usecases/ExportTranslationsUseCase.ts b/src/domain/usecases/ExportTranslationsUseCase.ts new file mode 100644 index 00000000..30c82e24 --- /dev/null +++ b/src/domain/usecases/ExportTranslationsUseCase.ts @@ -0,0 +1,70 @@ +import _ from "lodash"; +import { Async } from "domain/entities/Async"; +import { Locale } from "domain/entities/Locale"; +import { MetadataRepository } from "domain/repositories/MetadataRepository"; +import { LocalesRepository } from "domain/repositories/LocalesRepository"; +import { ExportTranslationsRepository } from "domain/repositories/ExportTranslationsRepository"; +import { ModelTranslationsExport } from "domain/entities/ModelTranslationsExport"; +import { getPluralModel } from "data/dhis2-utils"; +import log from "utils/log"; + +export interface ModelSelection { + model: string; // singular or plural, as provided + fields: string[]; // translatable fields, e.g. ["name", "formName"] +} + +interface Options { + outputFile: string; + models: ModelSelection[]; + locales: string[]; // locale names, e.g. ["Spanish", "French"] + includeData: boolean; +} + +export class ExportTranslationsUseCase { + constructor( + private repositories: { + metadata: MetadataRepository; + locales: LocalesRepository; + exportTranslations: ExportTranslationsRepository; + } + ) {} + + async execute(options: Options): Async { + const { outputFile, models, includeData } = options; + const allLocales = await this.repositories.locales.get(); + const locales = this.resolveLocales(allLocales, options.locales); + + const sheets = await Promise.all( + models.map(async (selection): Promise => { + const model = getPluralModel(selection.model); + const objects = await this.repositories.metadata.getAllWithTranslations([model]); + + log.info(`${model}: ${objects.length} objects`); + + return { model, fields: selection.fields, locales, objects }; + }) + ); + + await this.repositories.exportTranslations.save({ outputFile, sheets, includeData }); + } + + /* Match requested locale names against DB locales, ignoring any " (...)" suffix and case + (mirrors the import matching). Example: "Spanish" matches "Spanish (Spain)". */ + private resolveLocales(dbLocales: Locale[], requestedNames: string[]): Locale[] { + const stripName = (name: string) => + name + .replace(/\s*\(.*\)$/, "") + .trim() + .toLowerCase(); + const localesByName = _.keyBy(dbLocales, locale => stripName(locale.name)); + + return _(requestedNames) + .map(name => { + const locale = localesByName[stripName(name)]; + if (!locale) log.warn(`Locale not found in DB: ${name}`); + return locale; + }) + .compact() + .value(); + } +} diff --git a/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts b/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts new file mode 100644 index 00000000..0531a1b9 --- /dev/null +++ b/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts @@ -0,0 +1,97 @@ +import { describe, expect, test, vi } from "vitest"; +import { ExportTranslationsUseCase } from "../ExportTranslationsUseCase"; +import { Locale } from "domain/entities/Locale"; +import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; +import { MetadataRepository } from "domain/repositories/MetadataRepository"; +import { LocalesRepository } from "domain/repositories/LocalesRepository"; +import { ExportTranslationsRepository } from "domain/repositories/ExportTranslationsRepository"; +import log from "utils/log"; + +const locales: Locale[] = [ + { id: "1", name: "Spanish (Spain)", locale: "es" }, + { id: "2", name: "French", locale: "fr" }, + { id: "3", name: "Arabic", locale: "ar" }, +]; + +describe("ExportTranslationsUseCase", () => { + test("resolves locales by name (ignoring suffix/case), in the requested order, skipping unknown", async () => { + const warn = vi.spyOn(log, "warn").mockImplementation(() => undefined); + const { useCase, exportTranslations } = buildUseCase(); + + await useCase.execute({ + outputFile: "out.xlsx", + models: [{ model: "dataElement", fields: ["name"] }], + locales: ["French", "spanish", "Klingon"], + includeData: false, + }); + + const { sheets } = exportTranslations.save.mock.calls[0][0]; + expect(sheets[0].locales.map((l: Locale) => l.locale)).toEqual(["fr", "es"]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("Klingon")); + }); + + test("pluralizes the requested model for both the fetch and the sheet", async () => { + const { useCase, metadata, exportTranslations } = buildUseCase(); + + await useCase.execute({ + outputFile: "out.xlsx", + models: [{ model: "dataElement", fields: ["name"] }], + locales: ["French"], + includeData: true, + }); + + expect(metadata.getAllWithTranslations).toHaveBeenCalledWith(["dataElements"]); + const { sheets } = exportTranslations.save.mock.calls[0][0]; + expect(sheets[0].model).toBe("dataElements"); + }); + + test("builds one sheet per model and passes outputFile/includeData through", async () => { + const { useCase, exportTranslations } = buildUseCase(); + + await useCase.execute({ + outputFile: "translations.xlsx", + models: [ + { model: "dataElements", fields: ["name", "formName"] }, + { model: "indicators", fields: ["name"] }, + ], + locales: ["French"], + includeData: true, + }); + + const options = exportTranslations.save.mock.calls[0][0]; + expect(options.outputFile).toBe("translations.xlsx"); + expect(options.includeData).toBe(true); + expect(options.sheets.map((s: { model: string }) => s.model)).toEqual([ + "dataElements", + "indicators", + ]); + expect(options.sheets[0].fields).toEqual(["name", "formName"]); + }); +}); + +function buildUseCase() { + const object: MetadataObjectWithTranslations = { + model: "dataElements", + id: "abc", + name: "Element", + code: undefined, + translations: [], + }; + + const metadata = { + getAllWithTranslations: vi.fn().mockResolvedValue([object]), + getPaginated: vi.fn(), + save: vi.fn(), + } as unknown as MetadataRepository; + + const localesRepo: LocalesRepository = { get: vi.fn().mockResolvedValue(locales) }; + const exportTranslations = { save: vi.fn().mockResolvedValue(undefined) }; + + const useCase = new ExportTranslationsUseCase({ + metadata, + locales: localesRepo, + exportTranslations: exportTranslations as unknown as ExportTranslationsRepository, + }); + + return { useCase, metadata, localesRepo, exportTranslations }; +} diff --git a/src/scripts/commands/__tests__/translations.spec.ts b/src/scripts/commands/__tests__/translations.spec.ts new file mode 100644 index 00000000..e54052a1 --- /dev/null +++ b/src/scripts/commands/__tests__/translations.spec.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from "vitest"; +import { parseModels, parseModelsOption } from "../translations"; + +describe("parseModels", () => { + test("parses models with per-model fields", () => { + expect(parseModels("dataElements[name,formName],indicators[name]")).toEqual([ + { model: "dataElements", fields: ["name", "formName"] }, + { model: "indicators", fields: ["name"] }, + ]); + }); + + test("trims whitespace and keeps models without fields as empty", () => { + expect(parseModels("dataElements[ name , formName ], indicators")).toEqual([ + { model: "dataElements", fields: ["name", "formName"] }, + { model: "indicators", fields: [] }, + ]); + }); +}); + +describe("parseModelsOption (compulsory fields)", () => { + test("returns the selections when every model has fields", () => { + expect(parseModelsOption("dataElements[name],indicators[name]")).toEqual([ + { model: "dataElements", fields: ["name"] }, + { model: "indicators", fields: ["name"] }, + ]); + }); + + test("throws naming the models missing fields", () => { + expect(() => parseModelsOption("dataElements[name],indicators")).toThrow(/indicators/); + }); + + test("throws when no models are provided", () => { + expect(() => parseModelsOption("")).toThrow(/No models/); + }); +}); diff --git a/src/scripts/commands/translations.ts b/src/scripts/commands/translations.ts index 9099d069..9bdde295 100644 --- a/src/scripts/commands/translations.ts +++ b/src/scripts/commands/translations.ts @@ -1,10 +1,12 @@ import _ from "lodash"; import log from "utils/log"; -import { command, string, subcommands, positional, flag, option, optional } from "cmd-ts"; -import { getApiUrlOption, getD2Api } from "scripts/common"; +import { command, string, subcommands, positional, flag, option, optional, Type } from "cmd-ts"; +import { getApiUrlOptions, getD2ApiFromArgs } from "scripts/common"; import { TranslateMetadataUseCase } from "domain/usecases/TranslateMetadataUseCase"; +import { ExportTranslationsUseCase, ModelSelection } from "domain/usecases/ExportTranslationsUseCase"; import { LocalesD2Repository } from "data/LocalesD2Repository"; import { ImportTranslationsRepositorySpreadsheetRepository } from "data/ImportTranslationsRepositorySpreadsheetRepository"; +import { ExportTranslationsSpreadsheetRepository } from "data/ExportTranslationsSpreadsheetRepository"; import { MetadataD2Repository } from "data/MetadataD2Repository"; export function getCommand() { @@ -12,7 +14,7 @@ export function getCommand() { name: "from-spreadsheet", description: "Create translations for metadata objects", args: { - url: getApiUrlOption({ long: "url" }), + ...getApiUrlOptions(), post: flag({ long: "post", description: "Post changes", @@ -29,7 +31,7 @@ export function getCommand() { }), }, handler: async args => { - const api = getD2Api(args.url); + const api = getD2ApiFromArgs(args); const repositories = { metadata: new MetadataD2Repository(api), @@ -44,8 +46,109 @@ export function getCommand() { }, }); + const translateToSpreadsheetCmd = command({ + name: "to-spreadsheet", + description: "Generate a translations spreadsheet from metadata objects", + args: { + ...getApiUrlOptions(), + models: option({ + type: ModelsType, + long: "models", + description: + "Models to export, comma-separated, each with its translatable fields: " + + "model[field1,field2]. Example: dataElements[name,formName],indicators[name]", + }), + locales: option({ + type: string, + long: "locales", + description: "Locales to include as columns, comma-separated. Example: Spanish,French", + }), + includeData: flag({ + long: "include-data", + description: + "Write one row per object with source values and existing translations. " + + "When omitted, only the header row is written (a column template).", + }), + outputFile: positional({ + type: string, + displayName: "OUTPUT_XLSX_PATH", + description: "Output xlsx file", + }), + }, + handler: async args => { + const api = getD2ApiFromArgs(args); + + const repositories = { + metadata: new MetadataD2Repository(api), + locales: new LocalesD2Repository(api), + exportTranslations: new ExportTranslationsSpreadsheetRepository(), + }; + + await new ExportTranslationsUseCase(repositories).execute({ + outputFile: args.outputFile, + models: args.models, + locales: parseList(args.locales), + includeData: args.includeData, + }); + }, + }); + return subcommands({ name: "translations", - cmds: { "from-spreadsheet": translateFromSpreadsheetCmd }, + cmds: { + "from-spreadsheet": translateFromSpreadsheetCmd, + "to-spreadsheet": translateToSpreadsheetCmd, + }, }); } + +/* Parse a comma-separated list, trimming and dropping empty values. */ +function parseList(input: string): string[] { + return _(input.split(",")) + .map(value => value.trim()) + .compact() + .value(); +} + +/* Parse "dataElements[name,formName],indicators[name]" into [{ model, fields }, ...]. */ +export function parseModels(input: string): ModelSelection[] { + const regex = /([a-zA-Z][\w]*)(?:\[([^\]]*)\])?/g; + + return _(Array.from(input.matchAll(regex))) + .map(match => { + const model = match[1]; + if (!model) return undefined; + const fields = _((match[2] ?? "").split(",")) + .map(field => field.trim()) + .compact() + .value(); + return { model, fields }; + }) + .compact() + .value(); +} + +/* Parse and validate the --models option: every model must specify its translatable fields. */ +export function parseModelsOption(input: string): ModelSelection[] { + const selections = parseModels(input); + + if (selections.length === 0) throw new Error("No models provided"); + + const withoutFields = selections.filter(selection => _.isEmpty(selection.fields)); + if (!_.isEmpty(withoutFields)) { + const models = withoutFields.map(selection => selection.model).join(", "); + throw new Error( + `Missing translatable fields for: ${models}. ` + + `Specify them as model[field1,field2], e.g. indicators[name,shortName]` + ); + } + + return selections; +} + +/* cmd-ts type for --models: parses the string and requires every model to specify its fields. */ +const ModelsType: Type = { + async from(input) { + return parseModelsOption(input); + }, +}; diff --git a/yarn.lock b/yarn.lock index 518b3541..cc3e0e86 100644 --- a/yarn.lock +++ b/yarn.lock @@ -819,6 +819,14 @@ acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.7.0.tgz#90951fde0f8f09df93549481e5fc141445b791cf" integrity sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ== +adler-32@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.2.0.tgz#6a3e6bf0a63900ba15652808cb15c6813d1a5f25" + integrity sha512-/vUqU/UY4MVeFsg+SsK6c+/05RZXIHZMGJA+PX5JyWI0ZRcBpupnRuPLU/NXXoFwMYCPCoxIfElM2eS+DUXCqQ== + dependencies: + exit-on-epipe "~1.0.1" + printj "~1.1.0" + adler-32@~1.3.0: version "1.3.1" resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2" @@ -1091,7 +1099,7 @@ caniuse-lite@^1.0.30001317: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001317.tgz#0548fb28fd5bc259a70b8c1ffdbe598037666a1b" integrity sha512-xIZLh8gBm4dqNX0gkzrBeyI86J2eCjWzYAs40q88smG844YIrN4tVQl/RhquHvKEKImWWFIVh1Lxe5n1G/N+GQ== -cfb@~1.2.1: +cfb@^1.1.4, cfb@~1.2.1: version "1.2.2" resolved "https://registry.yarnpkg.com/cfb/-/cfb-1.2.2.tgz#94e687628c700e5155436dac05f74e08df23bc44" integrity sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA== @@ -1241,6 +1249,14 @@ cmd-ts@^0.10.0: didyoumean "^1.2.1" strip-ansi "^6.0.0" +codepage@~1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.14.0.tgz#8cbe25481323559d7d307571b0fff91e7a1d2f99" + integrity sha512-iz3zJLhlrg37/gYRWgEPkaFTtzmnEv1h+r7NgZum2lFElYQPi0/5bnmuDfODHxfp0INEfnRqyfyeIJDbb7ahRw== + dependencies: + commander "~2.14.1" + exit-on-epipe "~1.0.1" + codepage@~1.15.0: version "1.15.0" resolved "https://registry.yarnpkg.com/codepage/-/codepage-1.15.0.tgz#2e00519024b39424ec66eeb3ec07227e692618ab" @@ -1297,6 +1313,16 @@ commander@^8.0.0: resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66" integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww== +commander@~2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.14.1.tgz#2235123e37af8ca3c65df45b026dbd357b01b9aa" + integrity sha512-+YR16o3rK53SmWHU3rEM3tPAh2rwb1yPcQX5irVn7mb0gXbwuCCrnkbV5+PBfETdfg1vui07nM6PCG1zndcjQw== + +commander@~2.17.1: + version "2.17.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf" + integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg== + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -2047,6 +2073,11 @@ execa@^5.0.0: signal-exit "^3.0.3" strip-final-newline "^2.0.0" +exit-on-epipe@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692" + integrity sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw== + ext@^1.1.2: version "1.6.0" resolved "https://registry.yarnpkg.com/ext/-/ext-1.6.0.tgz#3871d50641e874cc172e2b53f919842d19db4c52" @@ -2125,6 +2156,16 @@ fdir@^6.4.4: resolved "https://registry.yarnpkg.com/fdir/-/fdir-6.4.4.tgz#1cfcf86f875a883e19a8fab53622cfe992e8d2f9" integrity sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg== +fflate@^0.3.8: + version "0.3.11" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.3.11.tgz#2c440d7180fdeb819e64898d8858af327b042a5d" + integrity sha512-Rr5QlUeGN1mbOHlaqcSYMKVpPbgLy0AWT/W0EHxA6NGI12yO1jpoui2zBBvU2G824ltM6Ut8BFgfHSBGfkmS0A== + +fflate@^0.8.3: + version "0.8.3" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.3.tgz#bc27d8eb30343d4d512abb03480202ce65d825fc" + integrity sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA== + file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" @@ -3495,6 +3536,11 @@ pretty-format@^29.5.0: ansi-styles "^5.0.0" react-is "^18.0.0" +printj@~1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/printj/-/printj-1.1.2.tgz#d90deb2975a8b9f600fb3a1c94e3f4c53c78a222" + integrity sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ== + prop-types@^15.6.2: version "15.8.1" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5" @@ -4786,6 +4832,22 @@ xdg-basedir@^4.0.0: resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" integrity sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q== +xlsx-js-style@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/xlsx-js-style/-/xlsx-js-style-1.2.0.tgz#58455f2fd3c5e22807c2841f5b0631a07098b719" + integrity sha512-DDT4FXFSWfT4DXMSok/m3TvmP1gvO3dn0Eu/c+eXHW5Kzmp7IczNkxg/iEPnImbG9X0Vb8QhROda5eatSR/97Q== + dependencies: + adler-32 "~1.2.0" + cfb "^1.1.4" + codepage "~1.14.0" + commander "~2.17.1" + crc-32 "~1.2.0" + exit-on-epipe "~1.0.1" + fflate "^0.3.8" + ssf "~0.11.2" + wmf "~1.0.1" + word "~0.3.0" + xlsx@^0.18.5: version "0.18.5" resolved "https://registry.yarnpkg.com/xlsx/-/xlsx-0.18.5.tgz#16711b9113c848076b8a177022799ad356eba7d0"