diff --git a/README.md b/README.md index 31a48990..ebe5db85 100644 --- a/README.md +++ b/README.md @@ -282,12 +282,32 @@ $ yarn start translations to-spreadsheet \ translations.xlsx ``` +Export only the objects of a metadata package that are new or renamed with respect to the instance, restricted to a translated data set (the typical "what do translators need for this feature" sheet): + +```shell +$ yarn start translations to-spreadsheet \ + --url='http://USER:PASSWORD@HOST:PORT' \ + --metadata-file=feature-metadata.json \ + --only-changed \ + --default-locale=en \ + --exclude-names='^\[DEPRECATED\]' \ + --data-set-ids=NQOwInnRDNL \ + --models='dataElements[formName],indicators[name]' \ + --locales='Spanish,French' \ + --include-data \ + translations.xlsx +``` + Notes: - `--models`: comma-separated list of models to export. Each model must specify its translatable fields with `[field1,field2]` (e.g. `indicators[name,shortName]`); a model without fields raises an error. - `--locales`: comma-separated list of locale names to include as columns, in the order given. The match ignores any ` (...)` suffix, so `Spanish` matches a `Spanish (Spain)` locale. - `--include-data`: write one row per object with the source values and the existing translations. When omitted, only the header row is written (a column template). -- `--program-id=ID` / `--data-set-id=ID` (exclusive): scope the export to the objects in that program's or data set's metadata dependency export (`/api/programs/{id}/metadata`, `/api/dataSets/{id}/metadata`) instead of the whole instance. Only the requested `--models` are kept from the export. +- `--program-ids=ID1,ID2` / `--data-set-ids=ID1,ID2` (exclusive): scope the export to the objects in those programs' or data sets' metadata dependency exports (`/api/programs/{id}/metadata`, `/api/dataSets/{id}/metadata`) instead of the whole instance. Only the requested `--models` are kept from the exports; an object shared by several parents appears once. With `--metadata-file`, `--data-set-ids` keeps the objects of the file belonging to those data sets (their data elements, indicators, sections and the options of those data elements). +- `--metadata-file`: read the objects from a DHIS2 metadata JSON export (`{"dataElements": [...], ...}`) instead of the instance. Useful when the metadata to translate is not yet deployed to a trusted instance. `--url` is still used to get the locales and as the reference for `--only-changed`. +- `--only-changed`: export only the objects of `--metadata-file` that need (re)translation: those not existing in the instance, or whose selected fields differ, or whose `--default-locale` translation of a selected field differs (a label can be changed only through that translation). Unselected fields (like a `name` prefix) are ignored. +- `--default-locale`: locale code of the default (DB) language, e.g. `en`. Matched by language, so `en` also matches `en_GB`. +- `--exclude-names`: regex; objects whose `name` matches are skipped. Example: `'^\[DEPRECATED\]'`. - One sheet (tab) is generated per model type. - Columns: `Type`, `UID`, then a group per field: the base source column `` followed by one `: ` column per selected locale. - Each field group is color-coded (bold colored header, source column highlighted, translation cells lightly tinted) so the grid is easy to scan. The header row and the first three columns (`Type`, `UID` and the first source column) are frozen, and columns within a field group share the same width. diff --git a/openspec/changes/export-translations-only-changed/.openspec.yaml b/openspec/changes/export-translations-only-changed/.openspec.yaml new file mode 100644 index 00000000..f2cbbe6a --- /dev/null +++ b/openspec/changes/export-translations-only-changed/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-09-18 diff --git a/openspec/changes/export-translations-only-changed/design.md b/openspec/changes/export-translations-only-changed/design.md new file mode 100644 index 00000000..5607390d --- /dev/null +++ b/openspec/changes/export-translations-only-changed/design.md @@ -0,0 +1,68 @@ +## Context + +`translations to-spreadsheet` (`ExportTranslationsUseCase`) fetches all objects of each model +through `MetadataRepository.getAllWithTranslations` and hands one `ModelTranslationsExport` per +model to `ExportTranslationsSpreadsheetRepository`. The source of the objects and the reference +instance were the same thing. + +## Decisions + +### Decision: Separate "source of objects" from "reference instance" + +A narrow `MetadataSourceRepository` interface (`getAllWithTranslations` only) is the source of the +objects to export; `MetadataRepository` extends it, so the instance keeps being the default source. +`MetadataJsonFileRepository` (data layer) implements the source from a metadata JSON export: it +pluralizes the requested models, tags each object with its `model`, defaults `translations` to `[]` +and keeps every other field so source columns (`formName`, ...) can be filled. + +The use case receives an optional `metadataSource`; the command handler builds it from +`--metadata-file`. Locales and the reference objects always come from the instance (`--url`). + +### Decision: Reference lookup by id, chunked + +`MetadataRepository.getByIdsWithTranslations(model, ids)` fetches only the file's ids from the +instance (`/api/metadata?model:fields=:owner&model:filter=id:in:[...]`, chunks of 100) instead of +downloading the whole model. Objects missing from the response are "new". + +### Decision: Change detection is a pure domain function + +`isChanged(object, reference, fields, defaultLocale)` compares, for the selected fields only: +the trimmed field value and (when a default locale is given) the trimmed translation of that field +in the default locale. Non-selected fields are ignored on purpose: a `[DEPRECATED]` prefix on +`name` does not require re-translating `formName`. The default-locale comparison covers labels +that are changed only through the `en` translation, which is how some projects keep a long English +label separate from the short `formName`. + +### Decision: Data set scope of a file resolved by membership + +`--data-set-ids` is one option with one meaning ("the objects of these data sets") and two +implementations behind `MetadataSourceRepository.getAllWithTranslations(models, { dataSetIds })`: +the instance merges the dependency exports of each id (#106, generalized to a list, deduplicated +by model+id); the file, which has no such export, builds a `DataSetScope` (domain entity) from +its own data sets and data elements: `dataSetElements` give +the data elements, the data set `indicators` give the indicators, the scoped data elements' +`optionSet` refs give the option sets; sections match through their `dataSet` ref. +`isInDataSetScope` dispatches on the object's model and rejects models with no data set relation, +so a wrong `--models` fails fast. `--program-ids` with a file is rejected. + +### Decision: Java legacy locale codes + +DHIS2 stores Indonesian translations with locale `in` while `/api/locales/db` reports `id` +(same for `iw`/`he`, `ji`/`yi`). `normalizeLocaleCode` in the `Locale` entity maps legacy codes +and `isSameLocale`/`haveSameLanguage` use it; the spreadsheet repository and the change detection +match translations through it, so existing Indonesian translations appear in their column. + +## Data flow + +CLI args → `MetadataJsonFileRepository` (file, scoped by `dataSetIds`) + `MetadataD2Repository`/ +`LocalesD2Repository` (instance) → `ExportTranslationsUseCase.getObjects` (exclude by name → +fetch reference by ids → `isChanged`) → `ModelTranslationsExport[]` → +`ExportTranslationsSpreadsheetRepository.save`. + +## Risks / Trade-offs + +- The file's own translations are what the sheet shows as "existing" translations. They are + expected to have been exported from the reference instance; otherwise translators see stale + values (they are still re-translating the row anyway). +- `--only-changed` without `--metadata-file` is rejected: comparing the instance with itself + yields an empty sheet. diff --git a/openspec/changes/export-translations-only-changed/proposal.md b/openspec/changes/export-translations-only-changed/proposal.md new file mode 100644 index 00000000..754b5146 --- /dev/null +++ b/openspec/changes/export-translations-only-changed/proposal.md @@ -0,0 +1,49 @@ +## Why + +When a feature ships new or renamed metadata (e.g. the CPR "Flex" package), translators need a +spreadsheet with just those objects, not the thousands already translated. Today +`translations to-spreadsheet` exports every object of a model from an instance, and the new +metadata may only exist as a JSON package (or in an instance that also carries unrelated work), +so the delta had to be computed by hand with ad-hoc scripts. + +## What Changes + +```sh +yarn start translations to-spreadsheet \ + --url=REFERENCE_INSTANCE \ + --metadata-file=feature-metadata.json \ + --only-changed \ + --default-locale=en \ + --exclude-names='^\[DEPRECATED\]' \ + --data-set-ids=DS1,DS2 \ + --models='dataElements[formName],indicators[name]' \ + --locales=... --include-data out.xlsx +``` + +- `--metadata-file=PATH`: objects (with their translations) are read from a DHIS2 metadata JSON + export instead of the instance. `--url` remains required: locales come from the instance, and + it is the reference for `--only-changed`. +- `--only-changed`: keep only objects that do not exist in the instance, or whose selected fields + differ, or whose `--default-locale` translation of a selected field differs. Requires + `--metadata-file`. +- `--default-locale=CODE`: language matched (`en` ~ `en_GB`), same semantics as the import side. +- `--exclude-names=REGEX`: skip objects whose `name` matches. +- `--program-id`/`--data-set-id` (from #106) become `--program-ids`/`--data-set-ids`, taking + comma-separated IDs (plural naming as in the other commands); the dependency exports of all + the IDs are merged. `--data-set-ids` also applies to `--metadata-file`: the file's objects are + kept by membership (data elements, indicators, sections, and the options of the data + elements' option sets), since a file has no dependency export. `--program-ids` is not + supported for a file. +- Translations stored with a Java legacy language code (`in` for Indonesian) are now matched to + the DB locale (`id`) when filling the existing-translation columns. + +Existing invocations are unaffected: without the new options the behavior is unchanged. + +Builds on #106 (`--program-id`/`--data-set-id`, short locale references, `Name` column). Old vs +new interface: `--program-id=ID` → `--program-ids=ID1,ID2`, `--data-set-id=ID` → +`--data-set-ids=ID1,ID2` (#106 is unmerged, so no released interface changes). + +## Non-goals + +- Diffing against a second instance (reference is always `--url`). +- Filtering instance exports by DHIS2 filter expressions; the delta case is served by the file. diff --git a/openspec/changes/export-translations-only-changed/specs/export-translations-to-spreadsheet/spec.md b/openspec/changes/export-translations-only-changed/specs/export-translations-to-spreadsheet/spec.md new file mode 100644 index 00000000..ec390374 --- /dev/null +++ b/openspec/changes/export-translations-only-changed/specs/export-translations-to-spreadsheet/spec.md @@ -0,0 +1,91 @@ +## ADDED Requirements + +### Requirement: Export objects from a metadata JSON file + +The `translations to-spreadsheet` command SHALL accept `--metadata-file=PATH`, a DHIS2 metadata +JSON export (`{"dataElements": [...], ...}`), and take the objects to export (with their +`translations`) from it instead of from the instance. Models are matched by their plural key. The +instance at `--url` SHALL still provide the locales. + +#### Scenario: Objects come from the file + +- **WHEN** the command runs with `--metadata-file=pkg.json --models='dataElements[formName]'` +- **THEN** the `dataElements` sheet lists the file's data elements, with their `formName` and the + translations present in the file, and no data element is read from the instance + +### Requirement: Export only the objects that need translation + +With `--only-changed`, the command SHALL keep an object only when it does not exist in the +instance, or when any selected field differs from the instance value, or when the +`--default-locale` translation of a selected field differs. Values are compared trimmed; fields +not selected in `--models` SHALL be ignored. `--only-changed` SHALL require `--metadata-file`. + +#### Scenario: Renamed label is exported, deprecation prefix is not + +- **WHEN** the file has data element A with `formName` changed and data element B whose only + change is a `[DEPRECATED]` prefix on `name`, with `--models='dataElements[formName]'` +- **THEN** A is in the sheet and B is not + +#### Scenario: Label changed only through the default-locale translation + +- **WHEN** a data element keeps its `formName` but its `en` `FORM_NAME` translation differs from + the instance, and the command runs with `--default-locale=en` +- **THEN** the data element is in the sheet + +#### Scenario: Only-changed without a file is rejected + +- **WHEN** the command runs with `--only-changed` and no `--metadata-file` +- **THEN** it exits with an error naming both options + +### Requirement: Exclude objects by name + +The command SHALL accept `--exclude-names=REGEX` and skip any object whose `name` matches it, +before change detection. + +#### Scenario: Deprecated indicators are skipped + +- **WHEN** the command runs with `--exclude-names='^\[DEPRECATED\]'` and `indicators[name]` +- **THEN** indicators whose name starts with `[DEPRECATED]` are not in the sheet even though their + `name` changed + +### Requirement: Scope options take several ids + +`--program-ids` and `--data-set-ids` SHALL accept comma-separated ids. Against an instance, the +metadata dependency exports of every id SHALL be merged, and an object present in several of +them SHALL appear once. + +#### Scenario: Two data sets sharing a data element + +- **WHEN** the command runs with `--data-set-ids=DS1,DS2` and both data sets contain data + element A +- **THEN** A appears once in the `dataElements` sheet + +### Requirement: Data set scope applies to a metadata file + +With `--metadata-file`, `--data-set-ids=ID1,ID2` SHALL keep only the file objects belonging to +those data sets: data elements listed in their `dataSetElements`, indicators listed in the data +sets, sections whose `dataSet` is one of them, and options of the option sets used by those data +elements. A data set id not found in the file, a requested model with no data set relation, or +`--program-ids` with a file SHALL abort with an error. + +#### Scenario: Data element of another form is not exported + +- **WHEN** the file has a new data element that belongs only to a data set not in + `--data-set-ids` +- **THEN** it is not in the sheet, even though it is new + +#### Scenario: Options follow their data element + +- **WHEN** a scoped data element uses an option set with new options +- **THEN** those options are in the `options` sheet + +### Requirement: Match translations stored with Java legacy locale codes + +Translation columns SHALL match an object's translations by locale ignoring the Java legacy +language code difference (`in`/`id`, `iw`/`he`, `ji`/`yi`). + +#### Scenario: Indonesian translation is shown + +- **WHEN** an object has a `FORM_NAME` translation with locale `in` and `Indonesian` (`id`) is a + requested locale +- **THEN** the `formName: Indonesian` cell holds that translation diff --git a/openspec/changes/export-translations-only-changed/tasks.md b/openspec/changes/export-translations-only-changed/tasks.md new file mode 100644 index 00000000..c5dc10ca --- /dev/null +++ b/openspec/changes/export-translations-only-changed/tasks.md @@ -0,0 +1,36 @@ +## 1. Domain layer + +- [x] `Locale`: `normalizeLocaleCode` (Java legacy codes), `isSameLocale`, `haveSameLanguage` +- [x] `MetadataObject`: `getMetadataObjectField`, `getMetadataObjectTranslation` (shared by the + spreadsheet repository and the change detection) +- [x] `MetadataSourceRepository` interface (with the #106 scope options); `MetadataRepository` + extends it and adds `getByIdsWithTranslations(model, ids)` +- [x] `DataSetScope`: `buildDataSetScope`, `isInDataSetScope` +- [x] `ExportTranslationsUseCase`: optional `metadataSource`, options `onlyChanged`, + `defaultLocale`, `excludeNames`; pure `isChanged` + +## 2. Data layer + +- [x] `MetadataJsonFileRepository` reading a DHIS2 metadata JSON export, `dataSetIds` scope by + membership +- [x] `MetadataD2Repository`: merge the dependency exports of several program/data set ids +- [x] `MetadataD2Repository.getByIdsWithTranslations` with chunked `id:in` filter +- [x] `ExportTranslationsSpreadsheetRepository` matches locales through `isSameLocale` + +## 3. Command wiring + +- [x] `--metadata-file`, `--only-changed`, `--default-locale`, `--exclude-names` on + `translations to-spreadsheet`; reject `--only-changed` without `--metadata-file` +- [x] `--program-id`/`--data-set-id` → `--program-ids`/`--data-set-ids` (comma-separated) + +## 4. Testing + +- [x] Use case: file source, exclude by name, only-changed selection, `isChanged` cases +- [x] `MetadataJsonFileRepository` (incl. data set scope), `DataSetScope` and `Locale` unit + tests; `in`/`id` match in the sheet + +## 5. Verification + +- [x] `yarn typecheck`, `yarn lint`, `yarn test` +- [x] README: document the options with a delta-export example +- [x] Spec under `specs/export-translations-to-spreadsheet` diff --git a/src/data/ExportTranslationsSpreadsheetRepository.ts b/src/data/ExportTranslationsSpreadsheetRepository.ts index 2404b325..00d12ee4 100644 --- a/src/data/ExportTranslationsSpreadsheetRepository.ts +++ b/src/data/ExportTranslationsSpreadsheetRepository.ts @@ -4,9 +4,12 @@ import XLSX from "xlsx-js-style"; import { unzipSync, zipSync } from "fflate"; import { Async } from "domain/entities/Async"; import { Locale } from "domain/entities/Locale"; -import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; +import { + getMetadataObjectField, + getMetadataObjectTranslation, + MetadataObjectWithTranslations, +} from "domain/entities/MetadataObject"; import { ModelTranslationsExport } from "domain/entities/ModelTranslationsExport"; -import { translationFieldToProperty } from "domain/entities/Translation"; import { ExportTranslationsOptions, ExportTranslationsRepository, @@ -176,7 +179,7 @@ export class ExportTranslationsSpreadsheetRepository implements ExportTranslatio private getRow(object: MetadataObjectWithTranslations, sheet: ModelTranslationsExport): string[] { const fieldCells = sheet.fields.flatMap(field => [ - getFieldValue(object, field), + getMetadataObjectField(object, field), ...sheet.locales.map(locale => this.getTranslationValue(object, field, locale)), ]); @@ -192,21 +195,10 @@ export class ExportTranslationsSpreadsheetRepository implements ExportTranslatio field: string, locale: Locale ): string { - const property = translationFieldToProperty(field); - const translation = object.translations.find( - t => t.property === property && t.locale === locale.locale - ); - return translation?.value ?? ""; + return getMetadataObjectTranslation(object, field, locale.locale) ?? ""; } } -function getFieldValue(object: MetadataObjectWithTranslations, field: string): string { - // Objects are fetched with `:owner`, so they carry all owner fields at runtime even - // though the type only declares id/name/code/translations. - const value = (object as unknown as Record)[field]; - return typeof value === "string" ? value : ""; -} - export interface SheetData { name: string; header: string[]; diff --git a/src/data/MetadataD2Repository.ts b/src/data/MetadataD2Repository.ts index 922cf4f7..8c0f6a28 100644 --- a/src/data/MetadataD2Repository.ts +++ b/src/data/MetadataD2Repository.ts @@ -8,7 +8,7 @@ import { Payload, SaveOptions, } from "domain/repositories/MetadataRepository"; -import { getPluralModel, runMetadata } from "./dhis2-utils"; +import { getPluralModel, promiseMap, runMetadata } from "./dhis2-utils"; import log from "utils/log"; import { MetadataModel, @@ -27,15 +27,23 @@ export class MetadataD2Repository implements MetadataRepository { models: string[], options?: GetTranslationsOptions ): Async { - if (options?.programId) { - return this.getDependencyMetadataObjects(models, "programs", options.programId); - } else if (options?.dataSetId) { - return this.getDependencyMetadataObjects(models, "dataSets", options.dataSetId); + if (!_.isEmpty(options?.programIds)) { + return this.getDependencyMetadataObjects(models, "programs", options?.programIds ?? []); + } else if (!_.isEmpty(options?.dataSetIds)) { + return this.getDependencyMetadataObjects(models, "dataSets", options?.dataSetIds ?? []); } else { return this.getMetadataObjects(models); } } + async getByIdsWithTranslations(model: string, ids: Id[]): Async { + // Filter by id in chunks to keep the request URL within limits. + const objectsByChunk = await promiseMap(_.chunk(ids, 100), idsChunk => + this.getMetadataObjects([model], { filter: `id:in:[${idsChunk.join(",")}]` }) + ); + return _.flatten(objectsByChunk); + } + async save(objects: MetadataObject[], options: SaveOptions): Async<{ payload: Payload; stats: object }> { const payload = await this.mergeWithExistingObjects(objects); @@ -125,9 +133,15 @@ export class MetadataD2Repository implements MetadataRepository { return metadataToPost; } - private async getD2Metadata(models: string[]): Async { + private async getD2Metadata(models: string[], options: GetOptions = {}): Async { const params = _(models) - .map(model => [`${getPluralModel(model)}:fields`, ":owner"] as [string, string]) + .flatMap((model): Array<[string, string]> => { + const modelPlural = getPluralModel(model); + return _.compact([ + [`${modelPlural}:fields`, ":owner"], + options.filter ? [`${modelPlural}:filter`, options.filter] : undefined, + ]); + }) .fromPairs() .value(); @@ -143,24 +157,37 @@ export class MetadataD2Repository implements MetadataRepository { return _(metadata).values().flatten().value(); } - private async getMetadataObjects(models: string[]): Async { - const metadata = await this.getD2Metadata(models); + private async getMetadataObjects( + models: string[], + options: GetOptions = {} + ): Async { + const metadata = await this.getD2Metadata(models, options); return this.mapMetadataObjects(metadata); } - /* Get objects from a program/dataSet metadata dependency export, keeping only the requested - models. The export groups objects by plural model name (plus non-array keys like "system", - which _.pick drops since they are not among the requested models). */ + /* Get objects from the programs/dataSets metadata dependency exports, keeping only the + requested models. The export groups objects by plural model name (plus non-array keys like + "system", which _.pick drops since they are not among the requested models). An object + shared by several parents is returned once. */ private async getDependencyMetadataObjects( models: string[], parentModel: "programs" | "dataSets", - parentId: Id + parentIds: Id[] ): Async { - log.debug(`GET ${parentModel} metadata: ${parentId}`); - const metadata = await this.api.get(`/${parentModel}/${parentId}/metadata.json`).getData(); - const requestedModels = models.map(getPluralModel); - return this.mapMetadataObjects(_.pick(metadata, requestedModels)); + + const objectsByParent = await promiseMap(parentIds, async parentId => { + log.debug(`GET ${parentModel} metadata: ${parentId}`); + const metadata = await this.api + .get(`/${parentModel}/${parentId}/metadata.json`) + .getData(); + return this.mapMetadataObjects(_.pick(metadata, requestedModels)); + }); + + return _(objectsByParent) + .flatten() + .uniqBy(object => `${object.model}:${object.id}`) + .value(); } private mapMetadataObjects(metadata: Metadata): MetadataObjectWithTranslations[] { @@ -189,6 +216,10 @@ type Model = string; type Metadata = Record>; +interface GetOptions { + filter?: string; // DHIS2 filter expression, e.g. "id:in:[ID1,ID2]" +} + interface D2ObjectBase { id: Id; name: string; diff --git a/src/data/MetadataJsonFileRepository.ts b/src/data/MetadataJsonFileRepository.ts new file mode 100644 index 00000000..5fe4e5cc --- /dev/null +++ b/src/data/MetadataJsonFileRepository.ts @@ -0,0 +1,81 @@ +import _ from "lodash"; +import fs from "fs"; +import { Async } from "domain/entities/Async"; +import { MetadataModel, MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; +import { + GetTranslationsOptions, + MetadataSourceRepository, +} from "domain/repositories/MetadataSourceRepository"; +import { buildDataSetScope, isInDataSetScope } from "domain/entities/DataSetScope"; +import { getPluralModel } from "./dhis2-utils"; +import { Translation } from "domain/entities/Translation"; + +/* Metadata objects read from a DHIS2 metadata JSON export ({ dataElements: [...], ... }), + for example a package to be deployed that is not yet in any trusted instance. */ +export class MetadataJsonFileRepository implements MetadataSourceRepository { + constructor(private path: string) {} + + async getAllWithTranslations( + models: MetadataModel[], + options: GetTranslationsOptions = {} + ): Async { + if (!_.isEmpty(options.programIds)) + throw new Error("programIds is not supported for a metadata file"); + + const metadata = this.read(); + const objects = this.getObjects(metadata, models); + const dataSetIds = options.dataSetIds ?? []; + + if (_.isEmpty(dataSetIds)) { + return objects; + } else { + // A file has no dependency export: scope by membership, using its data sets. + const dataSets = this.getObjects(metadata, ["dataSets"]); + const dataElements = this.getObjects(metadata, ["dataElements"]); + const scope = buildDataSetScope(dataSetIds, dataSets, dataElements); + + const missingIds = _.difference( + dataSetIds, + dataSets.map(dataSet => dataSet.id) + ); + if (!_.isEmpty(missingIds)) + throw new Error(`Data sets not found in ${this.path}: ${missingIds.join(", ")}`); + + return objects.filter(object => isInDataSetScope(object, scope)); + } + } + + private getObjects(metadata: MetadataFile, models: MetadataModel[]): MetadataObjectWithTranslations[] { + return _(models) + .map(getPluralModel) + .flatMap(model => + (metadata[model] ?? []).map( + (object): MetadataObjectWithTranslations => ({ + ...object, + model: model, + code: object.code, + translations: object.translations ?? [], + }) + ) + ) + .value(); + } + + private read(): MetadataFile { + const contents = fs.readFileSync(this.path, "utf8"); + const json = JSON.parse(contents) as unknown; + + if (!_.isPlainObject(json)) throw new Error(`Not a metadata JSON object: ${this.path}`); + + return _.pickBy(json as Record, _.isArray) as MetadataFile; + } +} + +type MetadataFile = Record>; + +interface MetadataFileObject { + id: string; + name: string; + code?: string; + translations?: Translation[]; +} diff --git a/src/data/__tests__/ExportTranslationsSpreadsheetRepository.spec.ts b/src/data/__tests__/ExportTranslationsSpreadsheetRepository.spec.ts index 4f6510ad..e8e52fe7 100644 --- a/src/data/__tests__/ExportTranslationsSpreadsheetRepository.spec.ts +++ b/src/data/__tests__/ExportTranslationsSpreadsheetRepository.spec.ts @@ -54,6 +54,26 @@ describe("ExportTranslationsSpreadsheetRepository.buildSheet", () => { // Second object has no formName field at all -> base column blank. expect(rows[1]).toEqual(["dataElement", "id2", "Second", "Second", "", "", "", "", ""]); }); + + test("matches translations stored with a Java legacy locale code (Indonesian in/id)", () => { + const indonesian: Locale = { id: "3", name: "Indonesian", locale: "id" }; + const object: MetadataObjectWithTranslations = { + model: "dataElements", + id: "id1", + name: "Hello", + code: undefined, + translations: [{ property: "NAME", locale: "in", value: "Halo" }], + }; + const sheet: ModelTranslationsExport = { + model: "dataElements", + fields: ["name"], + locales: [indonesian], + objects: [object], + }; + + const { rows } = repository.buildSheet(sheet, true); + expect(rows[0]).toEqual(["dataElement", "id1", "Hello", "Hello", "Halo"]); + }); }); describe("ExportTranslationsSpreadsheetRepository.save (file round-trip)", () => { diff --git a/src/data/__tests__/MetadataJsonFileRepository.spec.ts b/src/data/__tests__/MetadataJsonFileRepository.spec.ts new file mode 100644 index 00000000..f93bec81 --- /dev/null +++ b/src/data/__tests__/MetadataJsonFileRepository.spec.ts @@ -0,0 +1,73 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { MetadataJsonFileRepository } from "../MetadataJsonFileRepository"; + +const file = path.join(os.tmpdir(), `metadata-${Date.now()}.json`); + +const metadata = { + system: { id: "abc", date: "2026-09-18" }, + dataSets: [ + { + id: "ds1", + name: "Data set 1", + dataSetElements: [{ dataSet: { id: "ds1" }, dataElement: { id: "de1" } }], + }, + ], + dataElements: [ + { id: "de1", name: "DE 1", formName: "Form 1", translations: [] }, + { id: "de2", name: "DE 2", code: "DE2" }, + ], + indicators: [ + { + id: "ind1", + name: "Indicator 1", + translations: [{ property: "NAME", locale: "fr", value: "Indicateur 1" }], + }, + ], +}; + +describe("MetadataJsonFileRepository", () => { + beforeAll(() => fs.writeFileSync(file, JSON.stringify(metadata))); + afterAll(() => fs.rmSync(file)); + + test("returns the objects of the requested models (pluralized), with model and translations", async () => { + const repository = new MetadataJsonFileRepository(file); + + const objects = await repository.getAllWithTranslations(["indicator", "dataElements"]); + + expect(objects.map(o => [o.model, o.id])).toEqual([ + ["indicators", "ind1"], + ["dataElements", "de1"], + ["dataElements", "de2"], + ]); + expect(objects[0]?.translations).toEqual([{ property: "NAME", locale: "fr", value: "Indicateur 1" }]); + expect(objects[2]).toMatchObject({ code: "DE2", translations: [] }); + // Owner fields are kept, so they can be exported as source values. + expect(objects[1]).toMatchObject({ formName: "Form 1" }); + }); + + test("with dataSetIds returns only the objects belonging to those data sets", async () => { + const repository = new MetadataJsonFileRepository(file); + + const objects = await repository.getAllWithTranslations(["dataElements"], { dataSetIds: ["ds1"] }); + expect(objects.map(o => o.id)).toEqual(["de1"]); + + await expect( + repository.getAllWithTranslations(["dataElements"], { dataSetIds: ["ds1", "missing"] }) + ).rejects.toThrow("Data sets not found"); + }); + + test("with programIds fails: a file has no dependency export", async () => { + const repository = new MetadataJsonFileRepository(file); + await expect( + repository.getAllWithTranslations(["dataElements"], { programIds: ["p1"] }) + ).rejects.toThrow("programIds"); + }); + + test("returns no objects for models absent from the file", async () => { + const repository = new MetadataJsonFileRepository(file); + expect(await repository.getAllWithTranslations(["options"])).toEqual([]); + }); +}); diff --git a/src/domain/entities/DataSetScope.ts b/src/domain/entities/DataSetScope.ts new file mode 100644 index 00000000..3de8eb67 --- /dev/null +++ b/src/domain/entities/DataSetScope.ts @@ -0,0 +1,71 @@ +import _ from "lodash"; +import { Id } from "./Base"; +import { MetadataObject } from "./MetadataObject"; + +/* The objects reachable from a set of data sets: the ones a data-entry form (and its users) + actually see. Used to restrict exports to the data sets that need translation. */ +export interface DataSetScope { + dataSetIds: Id[]; + dataElementIds: Id[]; + indicatorIds: Id[]; + optionSetIds: Id[]; +} + +export const dataSetScopeModels = ["dataSets", "dataElements", "indicators", "sections", "options"]; + +export function buildDataSetScope( + dataSetIds: Id[], + dataSets: MetadataObject[], + dataElements: MetadataObject[] +): DataSetScope { + const scopedDataSets = dataSets.filter(dataSet => dataSetIds.includes(dataSet.id)); + + const dataElementIds = _(scopedDataSets) + .flatMap(dataSet => getRefs(dataSet, "dataSetElements").map(dse => getRef(dse, "dataElement"))) + .compact() + .uniq() + .value(); + + const indicatorIds = _(scopedDataSets) + .flatMap(dataSet => getRefs(dataSet, "indicators").map(ref => ref.id)) + .compact() + .uniq() + .value(); + + const optionSetIds = _(dataElements) + .filter(dataElement => dataElementIds.includes(dataElement.id)) + .map(dataElement => getRef(dataElement, "optionSet")) + .compact() + .uniq() + .value(); + + return { dataSetIds, dataElementIds, indicatorIds, optionSetIds }; +} + +export function isInDataSetScope(object: MetadataObject, scope: DataSetScope): boolean { + switch (object.model) { + case "dataSets": + return scope.dataSetIds.includes(object.id); + case "dataElements": + return scope.dataElementIds.includes(object.id); + case "indicators": + return scope.indicatorIds.includes(object.id); + case "sections": + return _.some(scope.dataSetIds, id => id === getRef(object, "dataSet")); + case "options": + return _.some(scope.optionSetIds, id => id === getRef(object, "optionSet")); + default: + throw new Error(`Model not supported by the data sets scope: ${object.model}`); + } +} + +/* Objects carry their owner fields at runtime (see MetadataObject); read references from them. */ +function getRef(object: object, field: string): Id | undefined { + const value = (object as Record)[field]; + return _.isPlainObject(value) ? (value as { id?: Id }).id : undefined; +} + +function getRefs(object: object, field: string): Array<{ id?: Id }> { + const value = (object as Record)[field]; + return _.isArray(value) ? (value as Array<{ id?: Id }>) : []; +} diff --git a/src/domain/entities/Locale.ts b/src/domain/entities/Locale.ts index 3bc469e4..b53e9009 100644 --- a/src/domain/entities/Locale.ts +++ b/src/domain/entities/Locale.ts @@ -17,3 +17,24 @@ export function getLocaleInfo(locale: string): { const [language = "", country] = locale.split("_", 2); return { language, country }; } + +/* DHIS2 (Java) stores some languages with their legacy ISO 639 code while /api/locales/db + reports the current one: Indonesian is "in" in translations but "id" in the locales list. */ +const legacyLanguageCodes: Record = { in: "id", iw: "he", ji: "yi" }; + +export function normalizeLocaleCode(locale: LocaleCode): LocaleCode { + const { language, country } = getLocaleInfo(locale); + const currentLanguage = legacyLanguageCodes[language] ?? language; + return country ? `${currentLanguage}_${country}` : currentLanguage; +} + +/* Locales may be LANGUAGE or LANGUAGE_COUNTRY: compare only the language part, so "en" + also matches "en_GB". */ +export function haveSameLanguage(locale1: LocaleCode, locale2: LocaleCode): boolean { + const language = (locale: LocaleCode) => getLocaleInfo(normalizeLocaleCode(locale)).language; + return language(locale1) === language(locale2); +} + +export function isSameLocale(locale1: LocaleCode, locale2: LocaleCode): boolean { + return normalizeLocaleCode(locale1) === normalizeLocaleCode(locale2); +} diff --git a/src/domain/entities/MetadataObject.ts b/src/domain/entities/MetadataObject.ts index 47f5a521..22a95401 100644 --- a/src/domain/entities/MetadataObject.ts +++ b/src/domain/entities/MetadataObject.ts @@ -1,6 +1,7 @@ import { Maybe } from "utils/ts-utils"; import { Id } from "./Base"; -import { Translation } from "./Translation"; +import { isSameLocale, LocaleCode } from "./Locale"; +import { Translation, translationFieldToProperty } from "./Translation"; export type MetadataModel = string; // Ex: "dataElements". @@ -14,3 +15,23 @@ export interface MetadataObject { export interface MetadataObjectWithTranslations extends MetadataObject { translations: Translation[]; } + +/* Objects are fetched with `:owner` (or read from a metadata export), so they carry all owner + fields at runtime even though the type only declares id/name/code/translations. */ +export function getMetadataObjectField(object: MetadataObject, field: string): string { + const value = (object as unknown as Record)[field]; + return typeof value === "string" ? value : ""; +} + +/* Translation of a translatable field (e.g. "formName" -> property FORM_NAME) in a locale. */ +export function getMetadataObjectTranslation( + object: MetadataObjectWithTranslations, + field: string, + locale: LocaleCode +): Maybe { + const property = translationFieldToProperty(field); + const translation = object.translations.find( + t => t.property === property && isSameLocale(t.locale, locale) + ); + return translation?.value; +} diff --git a/src/domain/entities/__tests__/DataSetScope.spec.ts b/src/domain/entities/__tests__/DataSetScope.spec.ts new file mode 100644 index 00000000..c9d6bda3 --- /dev/null +++ b/src/domain/entities/__tests__/DataSetScope.spec.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "vitest"; +import { buildDataSetScope, isInDataSetScope } from "../DataSetScope"; +import { MetadataObject } from "../MetadataObject"; + +const dataSets = [ + obj("dataSets", "ds1", { + dataSetElements: [{ dataSet: { id: "ds1" }, dataElement: { id: "de1" } }], + indicators: [{ id: "ind1" }], + }), + obj("dataSets", "ds2", { + dataSetElements: [{ dataSet: { id: "ds2" }, dataElement: { id: "de2" } }], + indicators: null, + }), +]; + +const dataElements = [ + obj("dataElements", "de1", { optionSet: { id: "os1" } }), + obj("dataElements", "de2", { optionSet: { id: "os2" } }), +]; + +const scope = buildDataSetScope(["ds1"], dataSets, dataElements); + +describe("buildDataSetScope", () => { + test("collects the data elements, indicators and option sets of the given data sets", () => { + expect(scope).toEqual({ + dataSetIds: ["ds1"], + dataElementIds: ["de1"], + indicatorIds: ["ind1"], + optionSetIds: ["os1"], + }); + }); +}); + +describe("isInDataSetScope", () => { + test("keeps objects reachable from the scoped data sets, by model", () => { + expect(isInDataSetScope(obj("dataElements", "de1"), scope)).toBe(true); + expect(isInDataSetScope(obj("dataElements", "de2"), scope)).toBe(false); + expect(isInDataSetScope(obj("indicators", "ind1"), scope)).toBe(true); + expect(isInDataSetScope(obj("indicators", "ind2"), scope)).toBe(false); + expect(isInDataSetScope(obj("sections", "s1", { dataSet: { id: "ds1" } }), scope)).toBe(true); + expect(isInDataSetScope(obj("sections", "s2", { dataSet: { id: "ds2" } }), scope)).toBe(false); + expect(isInDataSetScope(obj("options", "o1", { optionSet: { id: "os1" } }), scope)).toBe(true); + expect(isInDataSetScope(obj("options", "o2", { optionSet: { id: "os2" } }), scope)).toBe(false); + expect(isInDataSetScope(obj("dataSets", "ds1"), scope)).toBe(true); + }); + + test("rejects models without a data set relation", () => { + expect(() => isInDataSetScope(obj("programs", "p1"), scope)).toThrow("programs"); + }); +}); + +function obj(model: string, id: string, fields: object = {}): MetadataObject { + return { model, id, name: id, code: undefined, ...fields }; +} diff --git a/src/domain/entities/__tests__/Locale.spec.ts b/src/domain/entities/__tests__/Locale.spec.ts new file mode 100644 index 00000000..5fde161e --- /dev/null +++ b/src/domain/entities/__tests__/Locale.spec.ts @@ -0,0 +1,24 @@ +import { describe, expect, test } from "vitest"; +import { haveSameLanguage, isSameLocale, normalizeLocaleCode } from "../Locale"; + +describe("normalizeLocaleCode", () => { + test("maps Java legacy language codes to the current ISO codes, keeping the country", () => { + expect(normalizeLocaleCode("in")).toBe("id"); + expect(normalizeLocaleCode("iw_IL")).toBe("he_IL"); + expect(normalizeLocaleCode("es_ES")).toBe("es_ES"); + }); +}); + +describe("isSameLocale", () => { + test("matches Indonesian stored as 'in' with the DB locale 'id'", () => { + expect(isSameLocale("in", "id")).toBe(true); + expect(isSameLocale("en", "en_GB")).toBe(false); + }); +}); + +describe("haveSameLanguage", () => { + test("compares only the language part", () => { + expect(haveSameLanguage("en", "en_GB")).toBe(true); + expect(haveSameLanguage("en", "es")).toBe(false); + }); +}); diff --git a/src/domain/repositories/MetadataRepository.ts b/src/domain/repositories/MetadataRepository.ts index 73521b38..bdadf69c 100644 --- a/src/domain/repositories/MetadataRepository.ts +++ b/src/domain/repositories/MetadataRepository.ts @@ -6,27 +6,23 @@ import { MetadataObjectWithTranslations, } from "domain/entities/MetadataObject"; import { Paginated } from "domain/entities/Pagination"; +import { GetTranslationsOptions, MetadataSourceRepository } from "./MetadataSourceRepository"; -export interface MetadataRepository { +export { GetTranslationsOptions }; + +export interface MetadataRepository extends MetadataSourceRepository { getPaginated(options: { model: MetadataModel; page: number }): Async>; getAllWithTranslations( models: MetadataModel[], options?: GetTranslationsOptions ): Async; + getByIdsWithTranslations(model: MetadataModel, ids: Id[]): Async; save( objects: Obj[], options: SaveOptions ): Async<{ payload: Payload; stats: object }>; } -/* When programId/dataSetId is set, objects are taken from that program's/data set's metadata - dependency export (/api/programs/{id}/metadata, /api/dataSets/{id}/metadata) instead of the - whole instance. */ -export interface GetTranslationsOptions { - programId?: Id; - dataSetId?: Id; -} - export type Payload = Record; export interface SaveOptions { diff --git a/src/domain/repositories/MetadataSourceRepository.ts b/src/domain/repositories/MetadataSourceRepository.ts new file mode 100644 index 00000000..06d15461 --- /dev/null +++ b/src/domain/repositories/MetadataSourceRepository.ts @@ -0,0 +1,20 @@ +import { Async } from "domain/entities/Async"; +import { Id } from "domain/entities/Base"; +import { MetadataModel, MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; + +/* Read-only source of metadata objects with their translations: a DHIS2 instance or a + metadata JSON export. */ +export interface MetadataSourceRepository { + getAllWithTranslations( + models: MetadataModel[], + options?: GetTranslationsOptions + ): Async; +} + +/* When programIds/dataSetIds is set, only the objects belonging to those programs/data sets are + returned: from their metadata dependency exports (/api/programs/{id}/metadata, + /api/dataSets/{id}/metadata) for an instance, by membership for a metadata file. */ +export interface GetTranslationsOptions { + programIds?: Id[]; + dataSetIds?: Id[]; +} diff --git a/src/domain/usecases/ExportTranslationsUseCase.ts b/src/domain/usecases/ExportTranslationsUseCase.ts index 263e6128..49ef9787 100644 --- a/src/domain/usecases/ExportTranslationsUseCase.ts +++ b/src/domain/usecases/ExportTranslationsUseCase.ts @@ -1,10 +1,17 @@ import _ from "lodash"; import { Async } from "domain/entities/Async"; -import { Locale } from "domain/entities/Locale"; +import { Id } from "domain/entities/Base"; +import { Locale, LocaleCode } from "domain/entities/Locale"; import { MetadataRepository } from "domain/repositories/MetadataRepository"; +import { MetadataSourceRepository } from "domain/repositories/MetadataSourceRepository"; import { LocalesRepository } from "domain/repositories/LocalesRepository"; import { ExportTranslationsRepository } from "domain/repositories/ExportTranslationsRepository"; import { ModelTranslationsExport } from "domain/entities/ModelTranslationsExport"; +import { + getMetadataObjectField, + getMetadataObjectTranslation, + MetadataObjectWithTranslations, +} from "domain/entities/MetadataObject"; import { getPluralModel } from "data/dhis2-utils"; import { Maybe } from "utils/ts-utils"; import log from "utils/log"; @@ -19,34 +26,34 @@ interface Options { models: ModelSelection[]; locales: string[]; // locale names, e.g. ["Spanish", "French"] includeData: boolean; - programId?: string; // when set, scope the export to this program's metadata dependency export - dataSetId?: string; // when set, scope the export to this data set's metadata dependency export + programIds?: Id[]; // when set, scope the export to these programs' objects + dataSetIds?: Id[]; // when set, scope the export to these data sets' objects + onlyChanged?: boolean; // keep only objects new or changed with respect to the instance + defaultLocale?: LocaleCode; // onlyChanged also compares this locale's translations (ex: "en") + excludeNames?: RegExp; // drop objects whose name matches (ex: /^\[DEPRECATED\]/) } export class ExportTranslationsUseCase { constructor( private repositories: { - metadata: MetadataRepository; + metadata: MetadataRepository; // the instance: locales and reference for onlyChanged + metadataSource?: MetadataSourceRepository; // objects to export (default: the instance) locales: LocalesRepository; exportTranslations: ExportTranslationsRepository; } ) {} async execute(options: Options): Async { - const { outputFile, models, includeData, programId, dataSetId } = options; - if (programId && dataSetId) throw new Error("Options programId and dataSetId are exclusive"); + const { outputFile, models, includeData, programIds, dataSetIds } = options; + if (!_.isEmpty(programIds) && !_.isEmpty(dataSetIds)) + throw new Error("Options programIds and dataSetIds are exclusive"); const allLocales = await this.repositories.locales.get(); const locales = this.resolveLocales(allLocales, options.locales); const sheets = await Promise.all( models.map(async (selection): Promise => { const model = getPluralModel(selection.model); - const objects = await this.repositories.metadata.getAllWithTranslations([model], { - programId, - dataSetId, - }); - - log.info(`${model}: ${objects.length} objects`); + const objects = await this.getObjects(model, selection.fields, options); return { model, fields: selection.fields, locales, objects }; }) @@ -55,6 +62,35 @@ export class ExportTranslationsUseCase { await this.repositories.exportTranslations.save({ outputFile, sheets, includeData }); } + private async getObjects( + model: string, + fields: string[], + options: Options + ): Async { + const { programIds, dataSetIds, excludeNames } = options; + const source = this.repositories.metadataSource ?? this.repositories.metadata; + const allObjects = await source.getAllWithTranslations([model], { programIds, dataSetIds }); + const objects = excludeNames + ? allObjects.filter(object => !excludeNames.test(object.name)) + : allObjects; + + if (!options.onlyChanged) { + log.info(`${model}: ${objects.length} objects`); + return objects; + } else { + const ids = objects.map(object => object.id); + const referenceObjects = await this.repositories.metadata.getByIdsWithTranslations(model, ids); + const referenceById = _.keyBy(referenceObjects, object => object.id); + + const changed = objects.filter(object => + isChanged(object, referenceById[object.id], fields, options.defaultLocale) + ); + + log.info(`${model}: ${objects.length} objects, ${changed.length} new or changed`); + return changed; + } + } + /* Resolve each requested name to a DB locale, allowing short references. The resolved locale's name is stripped of its " (...)" suffix so the column header round-trips with the import (which matches headers against suffix-stripped DB names). */ @@ -92,6 +128,29 @@ export class ExportTranslationsUseCase { } } +/* An object needs (re)translation when it does not exist in the reference, or when any of the + selected fields differs, or when the default-locale translation of a selected field differs + (the label users see may be changed only through that translation). */ +export function isChanged( + object: MetadataObjectWithTranslations, + reference: Maybe, + fields: string[], + defaultLocale: Maybe +): boolean { + if (!reference) return true; + + const fieldValue = (obj: MetadataObjectWithTranslations, field: string) => + getMetadataObjectField(obj, field).trim(); + const defaultTranslation = (obj: MetadataObjectWithTranslations, field: string) => + defaultLocale ? getMetadataObjectTranslation(obj, field, defaultLocale)?.trim() ?? "" : ""; + + return fields.some( + field => + fieldValue(object, field) !== fieldValue(reference, field) || + defaultTranslation(object, field) !== defaultTranslation(reference, field) + ); +} + /* Drop a trailing " (...)" country/variant qualifier, keeping the original case. */ function stripLocaleSuffix(name: string): string { return name.replace(/\s*\(.*\)$/, "").trim(); diff --git a/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts b/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts index 45772c89..b5a5bceb 100644 --- a/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts +++ b/src/domain/usecases/__tests__/ExportTranslationsUseCase.spec.ts @@ -1,10 +1,14 @@ import { describe, expect, test, vi } from "vitest"; -import { ExportTranslationsUseCase } from "../ExportTranslationsUseCase"; +import { ExportTranslationsUseCase, isChanged } from "../ExportTranslationsUseCase"; import { Locale } from "domain/entities/Locale"; import { MetadataObjectWithTranslations } from "domain/entities/MetadataObject"; import { MetadataRepository } from "domain/repositories/MetadataRepository"; import { LocalesRepository } from "domain/repositories/LocalesRepository"; -import { ExportTranslationsRepository } from "domain/repositories/ExportTranslationsRepository"; +import { + ExportTranslationsOptions, + ExportTranslationsRepository, +} from "domain/repositories/ExportTranslationsRepository"; +import { MetadataSourceRepository } from "domain/repositories/MetadataSourceRepository"; import log from "utils/log"; const locales: Locale[] = [ @@ -31,10 +35,12 @@ describe("ExportTranslationsUseCase", () => { }); test("resolves a short reference by substring and strips the suffix from the column name", async () => { - const { useCase, exportTranslations } = buildUseCase([ - { id: "1", name: "Southern Sotho (Lesotho)", locale: "st" }, - { id: "2", name: "Thai (Thailand)", locale: "th" }, - ]); + const { useCase, exportTranslations } = buildUseCase({ + locales: [ + { id: "1", name: "Southern Sotho (Lesotho)", locale: "st" }, + { id: "2", name: "Thai (Thailand)", locale: "th" }, + ], + }); await useCase.execute({ outputFile: "out.xlsx", @@ -49,10 +55,12 @@ describe("ExportTranslationsUseCase", () => { }); test("throws when a reference is ambiguous (matches more than one locale)", async () => { - const { useCase } = buildUseCase([ - { id: "1", name: "Norwegian Bokmål (Norway)", locale: "nb" }, - { id: "2", name: "Norwegian Nynorsk (Norway)", locale: "nn" }, - ]); + const { useCase } = buildUseCase({ + locales: [ + { id: "1", name: "Norwegian Bokmål (Norway)", locale: "nb" }, + { id: "2", name: "Norwegian Nynorsk (Norway)", locale: "nn" }, + ], + }); await expect( useCase.execute({ @@ -75,14 +83,14 @@ describe("ExportTranslationsUseCase", () => { }); expect(metadata.getAllWithTranslations).toHaveBeenCalledWith(["dataElements"], { - programId: undefined, - dataSetId: undefined, + programIds: undefined, + dataSetIds: undefined, }); const { sheets } = exportTranslations.save.mock.calls[0][0]; expect(sheets[0].model).toBe("dataElements"); }); - test("scopes the fetch to the given program when programId is set", async () => { + test("scopes the fetch to the given program when programIds is set", async () => { const { useCase, metadata } = buildUseCase(); await useCase.execute({ @@ -90,16 +98,16 @@ describe("ExportTranslationsUseCase", () => { models: [{ model: "dataElement", fields: ["name"] }], locales: ["French"], includeData: true, - programId: "PROG123", + programIds: ["PROG123"], }); expect(metadata.getAllWithTranslations).toHaveBeenCalledWith(["dataElements"], { - programId: "PROG123", - dataSetId: undefined, + programIds: ["PROG123"], + dataSetIds: undefined, }); }); - test("scopes the fetch to the given data set when dataSetId is set", async () => { + test("scopes the fetch to the given data set when dataSetIds is set", async () => { const { useCase, metadata } = buildUseCase(); await useCase.execute({ @@ -107,16 +115,16 @@ describe("ExportTranslationsUseCase", () => { models: [{ model: "dataElement", fields: ["formName"] }], locales: ["French"], includeData: true, - dataSetId: "DS123", + dataSetIds: ["DS123"], }); expect(metadata.getAllWithTranslations).toHaveBeenCalledWith(["dataElements"], { - programId: undefined, - dataSetId: "DS123", + programIds: undefined, + dataSetIds: ["DS123"], }); }); - test("rejects programId and dataSetId set at the same time", async () => { + test("rejects programIds and dataSetIds set at the same time", async () => { const { useCase } = buildUseCase(); await expect( @@ -125,12 +133,81 @@ describe("ExportTranslationsUseCase", () => { models: [{ model: "dataElement", fields: ["name"] }], locales: ["French"], includeData: true, - programId: "PROG123", - dataSetId: "DS123", + programIds: ["PROG123"], + dataSetIds: ["DS123"], }) ).rejects.toThrow(/exclusive/); }); + test("reads objects from metadataSource when given, passing the scope, not from the instance", async () => { + const fileObject = buildObject({ id: "file1", name: "From file" }); + const metadataSource = { getAllWithTranslations: vi.fn().mockResolvedValue([fileObject]) }; + const { useCase, metadata, exportTranslations } = buildUseCase({ metadataSource }); + + await useCase.execute({ + outputFile: "out.xlsx", + models: [{ model: "dataElements", fields: ["name"] }], + locales: ["French"], + includeData: true, + dataSetIds: ["ds1"], + }); + + expect(metadataSource.getAllWithTranslations).toHaveBeenCalledWith(["dataElements"], { + programIds: undefined, + dataSetIds: ["ds1"], + }); + expect(metadata.getAllWithTranslations).not.toHaveBeenCalled(); + expect(getExportedObjects(exportTranslations)).toEqual([fileObject]); + }); + + test("excludeNames drops objects whose name matches", async () => { + const kept = buildObject({ id: "id1", name: "Active" }); + const deprecated = buildObject({ id: "id2", name: "[DEPRECATED] Old" }); + const { useCase, exportTranslations } = buildUseCase({ objects: [kept, deprecated] }); + + await useCase.execute({ + outputFile: "out.xlsx", + models: [{ model: "dataElements", fields: ["name"] }], + locales: ["French"], + includeData: true, + excludeNames: /^\[DEPRECATED\]/, + }); + + expect(getExportedObjects(exportTranslations)).toEqual([kept]); + }); + + test("onlyChanged keeps new and changed objects, comparing against the instance by id", async () => { + const unchanged = buildObject({ id: "same", formName: "Same label" }); + const renamed = buildObject({ id: "renamed", formName: "New label" }); + const added = buildObject({ id: "added", formName: "Brand new" }); + const metadataSource = { + getAllWithTranslations: vi.fn().mockResolvedValue([unchanged, renamed, added]), + }; + const referenceObjects = [ + buildObject({ id: "same", formName: "Same label" }), + buildObject({ id: "renamed", formName: "Old label" }), + ]; + const { useCase, metadata, exportTranslations } = buildUseCase({ + metadataSource, + referenceObjects, + }); + + await useCase.execute({ + outputFile: "out.xlsx", + models: [{ model: "dataElements", fields: ["formName"] }], + locales: ["French"], + includeData: true, + onlyChanged: true, + }); + + expect(metadata.getByIdsWithTranslations).toHaveBeenCalledWith("dataElements", [ + "same", + "renamed", + "added", + ]); + expect(getExportedObjects(exportTranslations).map(o => o.id)).toEqual(["renamed", "added"]); + }); + test("builds one sheet per model and passes outputFile/includeData through", async () => { const { useCase, exportTranslations } = buildUseCase(); @@ -152,29 +229,102 @@ describe("ExportTranslationsUseCase", () => { }); }); -function buildUseCase(localesList: Locale[] = locales) { - const object: MetadataObjectWithTranslations = { - model: "dataElements", - id: "abc", - name: "Element", - code: undefined, - translations: [], - }; +describe("isChanged", () => { + const fields = ["formName"]; + + test("is true when the object does not exist in the reference", () => { + expect(isChanged(buildObject({ id: "new" }), undefined, fields, "en")).toBe(true); + }); + + test("is false when the selected fields are equal, ignoring surrounding whitespace", () => { + const object = buildObject({ id: "id1", formName: "Label " }); + const reference = buildObject({ id: "id1", formName: "Label" }); + expect(isChanged(object, reference, fields, undefined)).toBe(false); + }); + + test("ignores changes in non-selected fields", () => { + const object = buildObject({ id: "id1", name: "[DEPRECATED] Name", formName: "Label" }); + const reference = buildObject({ id: "id1", name: "Name", formName: "Label" }); + expect(isChanged(object, reference, fields, "en")).toBe(false); + }); + + test("is true when only the default-locale translation of a selected field changed", () => { + const object = buildObject({ + id: "id1", + formName: "Label", + translations: [{ property: "FORM_NAME", locale: "en_GB", value: "Label per person" }], + }); + const reference = buildObject({ + id: "id1", + formName: "Label", + translations: [{ property: "FORM_NAME", locale: "en", value: "Label" }], + }); + + expect(isChanged(object, reference, fields, "en")).toBe(true); + expect(isChanged(object, reference, fields, undefined)).toBe(false); + }); + + test("ignores translation changes in other locales", () => { + const object = buildObject({ + id: "id1", + translations: [{ property: "FORM_NAME", locale: "fr", value: "Nouveau" }], + }); + const reference = buildObject({ + id: "id1", + translations: [{ property: "FORM_NAME", locale: "fr", value: "Ancien" }], + }); + expect(isChanged(object, reference, fields, "en")).toBe(false); + }); +}); + +function buildUseCase( + options: { + locales?: Locale[]; + objects?: MetadataObjectWithTranslations[]; + referenceObjects?: MetadataObjectWithTranslations[]; + metadataSource?: MetadataSourceRepository; + } = {} +) { + const objects = options.objects ?? [buildObject({})]; const metadata = { - getAllWithTranslations: vi.fn().mockResolvedValue([object]), + getAllWithTranslations: vi.fn().mockResolvedValue(objects), + getByIdsWithTranslations: vi.fn().mockResolvedValue(options.referenceObjects ?? []), getPaginated: vi.fn(), save: vi.fn(), } as unknown as MetadataRepository; - const localesRepo: LocalesRepository = { get: vi.fn().mockResolvedValue(localesList) }; + const localesRepo: LocalesRepository = { + get: vi.fn().mockResolvedValue(options.locales ?? locales), + }; const exportTranslations = { save: vi.fn().mockResolvedValue(undefined) }; const useCase = new ExportTranslationsUseCase({ metadata, + metadataSource: options.metadataSource, locales: localesRepo, exportTranslations: exportTranslations as unknown as ExportTranslationsRepository, }); return { useCase, metadata, localesRepo, exportTranslations }; } + +function buildObject( + attrs: Partial & { formName?: string } +): MetadataObjectWithTranslations { + return { + model: "dataElements", + id: "abc", + name: "Element", + code: undefined, + translations: [], + ...attrs, + }; +} + +function getExportedObjects(exportTranslations: { + save: ReturnType; +}): MetadataObjectWithTranslations[] { + const options = exportTranslations.save.mock.calls[0]?.[0] as ExportTranslationsOptions; + return options.sheets[0]?.objects ?? []; +} diff --git a/src/scripts/commands/translations.ts b/src/scripts/commands/translations.ts index d7455b64..b6e37122 100644 --- a/src/scripts/commands/translations.ts +++ b/src/scripts/commands/translations.ts @@ -1,13 +1,14 @@ import _ from "lodash"; import log from "utils/log"; import { command, string, subcommands, positional, flag, option, optional, Type } from "cmd-ts"; -import { getApiUrlOptions, getD2ApiFromArgs } from "scripts/common"; +import { getApiUrlOptions, getD2ApiFromArgs, IdsSeparatedByCommas } from "scripts/common"; import { TranslateMetadataUseCase } from "domain/usecases/TranslateMetadataUseCase"; import { ExportTranslationsUseCase, ModelSelection } from "domain/usecases/ExportTranslationsUseCase"; import { LocalesD2Repository } from "data/LocalesD2Repository"; import { ImportTranslationsRepositorySpreadsheetRepository } from "data/ImportTranslationsRepositorySpreadsheetRepository"; import { ExportTranslationsSpreadsheetRepository } from "data/ExportTranslationsSpreadsheetRepository"; import { MetadataD2Repository } from "data/MetadataD2Repository"; +import { MetadataJsonFileRepository } from "data/MetadataJsonFileRepository"; export function getCommand() { const translateFromSpreadsheetCmd = command({ @@ -63,19 +64,20 @@ export function getCommand() { long: "locales", description: "Locales to include as columns, comma-separated. Example: Spanish,French", }), - programId: option({ - type: optional(string), - long: "program-id", + programIds: option({ + type: optional(IdsSeparatedByCommas), + long: "program-ids", description: - "Scope the export to a program's metadata dependency export " + - "(/api/programs/{id}/metadata) instead of the whole instance", + "Scope the export to the metadata dependency export of these programs " + + "(/api/programs/{id}/metadata, comma-separated IDs) instead of the whole instance", }), - dataSetId: option({ - type: optional(string), - long: "data-set-id", + dataSetIds: option({ + type: optional(IdsSeparatedByCommas), + long: "data-set-ids", description: - "Scope the export to a data set's metadata dependency export " + - "(/api/dataSets/{id}/metadata) instead of the whole instance", + "Scope the export to the metadata dependency export of these data sets " + + "(/api/dataSets/{id}/metadata, comma-separated IDs) instead of the whole " + + "instance. With --metadata-file, to the objects of the file belonging to them", }), includeData: flag({ long: "include-data", @@ -83,6 +85,31 @@ export function getCommand() { "Write one row per object with source values and existing translations. " + "When omitted, only the header row is written (a column template).", }), + metadataFile: option({ + type: optional(string), + long: "metadata-file", + description: + "Read the objects from a DHIS2 metadata JSON export instead of the instance " + + "(--url is still used for the locales and as reference for --only-changed)", + }), + onlyChanged: flag({ + long: "only-changed", + description: + "Export only the objects of --metadata-file that do not exist in the instance " + + "or whose selected fields (or their --default-locale translation) differ", + }), + defaultLocale: option({ + type: optional(string), + long: "default-locale", + description: + "Locale code of the default (DB) language, used by --only-changed to also " + + "detect changes in its translations. Matched by language. Example: en", + }), + excludeNames: option({ + type: optional(RegExpType), + long: "exclude-names", + description: "Skip objects whose name matches this regex. Example: '^\\[DEPRECATED\\]'", + }), outputFile: positional({ type: string, displayName: "OUTPUT_XLSX_PATH", @@ -92,8 +119,14 @@ export function getCommand() { handler: async args => { const api = getD2ApiFromArgs(args); + if (args.onlyChanged && !args.metadataFile) + throw new Error("--only-changed requires --metadata-file"); + const repositories = { metadata: new MetadataD2Repository(api), + metadataSource: args.metadataFile + ? new MetadataJsonFileRepository(args.metadataFile) + : undefined, locales: new LocalesD2Repository(api), exportTranslations: new ExportTranslationsSpreadsheetRepository(), }; @@ -103,8 +136,11 @@ export function getCommand() { models: args.models, locales: parseList(args.locales), includeData: args.includeData, - programId: args.programId, - dataSetId: args.dataSetId, + programIds: args.programIds, + dataSetIds: args.dataSetIds, + onlyChanged: args.onlyChanged, + defaultLocale: args.defaultLocale, + excludeNames: args.excludeNames, }); }, }); @@ -162,6 +198,13 @@ export function parseModelsOption(input: string): ModelSelection[] { return selections; } +/* cmd-ts type for regex options. */ +const RegExpType: Type = { + async from(input) { + return new RegExp(input); + }, +}; + /* cmd-ts type for --models: parses the string and requires every model to specify its fields. */ const ModelsType: Type = { async from(input) {