diff --git a/README.md b/README.md index 1f915109..d34f1c9b 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,16 @@ Notes: Export a program with all its associated metadata and data (events, enrollments, tracked entities). +The data can be filtered by: +- Organization Unit via the `--orgunits-ids` option. If `--descendants` is used, the included orgUnits will be the specified and its descendants. Note that if no `--orgunits-ids` is provided the `ouMode` will be `ALL` and the `--descendants` option will be redundant. +- The `lastUpdated` property using the `--updated-start-date` and/or `--updated-end-date` options. These options accept one of `YYYY-MM-DD`, `YYYY-MM-DDTHH:mm` or `YYYY-MM-DDTHH:mm:ss.sss`. A date (`YYYY-MM-DD`) defaults to the start of that day for `--updated-start-date` and the end of that day for `--updated-end-date` (e.g. `--updated-end-date=2026-01-01` is treated as `2026-01-01T23:59:59.999`). If a time is given, missing seconds/milliseconds default to zero, e.g. `2026-01-01T06:00` is treated as `2026-01-01T06:00:00.000` for both. These options apply the following criteria: `--updated-start-date` will include elements with `updatedAt` equal or greater, `--updated-end-date` will include elements with `updatedAt` strictly lesser than it. + +**Please note that the updated date options are applied independently to the three metadata types (events, enrollments, tracked entities). This can lead to the export referencing elements that are not in the export. For example events whose parent enrollment and/or TEI has an `updatedAt` date that falls outside of the date window** + +The metadata export can be skipped via the `--skip-metadata` option. + +The login details can be either provided as `--url='http://USER:PASSWORD@HOST:PORT'` or `--url='http://HOST:PORT' --auth='USER:PASSWORD'`. The `--auth` option is useful with complex passwords that can cause errors when used in the `--url` only approach. + ```shell $ yarn start programs export --url='http://USER:PASSWORD@HOST:PORT' \ --programs-ids=kX2GpLIa75l,kpNc7KvydVz programs.json diff --git a/src/data/D2Tracker.ts b/src/data/D2Tracker.ts index 06a3ba1f..db67aacc 100644 --- a/src/data/D2Tracker.ts +++ b/src/data/D2Tracker.ts @@ -76,12 +76,14 @@ export class D2Tracker { orgUnitIds: string[] | undefined; trackedEntity?: string | undefined; children?: boolean; + updatedStartDate?: string; + updatedEndDate?: string; } ): Promise> { type Output = Array; const output: Output = []; - const { programIds, orgUnitIds, trackedEntity } = options; + const { programIds, orgUnitIds, trackedEntity, updatedStartDate, updatedEndDate } = options; for (const programId of programIds) { let page = 1; @@ -96,19 +98,22 @@ export class D2Tracker { const apiOptions = { page: page, pageSize: pageSize, + // NOTE: For 2.41+ ouMode is orgUnitMode ouMode: ouMode as typeof ouMode, orgUnit: orgUnitIds?.join(";"), fields: { $all: true } as const, program: programId, trackedEntity, + updatedAfter: updatedStartDate, }; const { tracker } = this.api; const endpoint = { - trackedEntities: () => tracker.trackedEntities.get(apiOptions), + trackedEntities: () => + tracker.trackedEntities.get({ ...apiOptions, updatedBefore: updatedEndDate }), enrollments: () => tracker.enrollments.get(apiOptions), - events: () => tracker.events.get(apiOptions), + events: () => tracker.events.get({ ...apiOptions, updatedBefore: updatedEndDate }), }; const res = await endpoint[model]().getData(); @@ -117,7 +122,11 @@ export class D2Tracker { if (instances.length === 0) { dataRemaining = false; } else { - output.push(...instances); + const filtered = + model === "enrollments" + ? this.filterByUpdatedEndDate(instances, updatedEndDate) + : instances; + output.push(...filtered); page++; } } @@ -126,6 +135,16 @@ export class D2Tracker { return output; } + + // NOTE: the enrollments endpoint has no updatedBefore param + private filterByUpdatedEndDate( + instances: T[], + updatedEndDate: string | undefined + ): T[] { + if (!updatedEndDate) return instances; + const endDate = new Date(updatedEndDate).getTime(); + return instances.filter(instance => new Date(instance.updatedAt).getTime() < endDate); + } } type TrackerResponse = { diff --git a/src/data/ProgramsD2Repository.ts b/src/data/ProgramsD2Repository.ts index 12ca0be2..fa567d04 100644 --- a/src/data/ProgramsD2Repository.ts +++ b/src/data/ProgramsD2Repository.ts @@ -2,7 +2,7 @@ import _ from "lodash"; import { Async } from "domain/entities/Async"; import { Id } from "domain/entities/Base"; import { ProgramExport } from "domain/entities/ProgramExport"; -import { ProgramsRepository, RunRulesOptions } from "domain/repositories/ProgramsRepository"; +import { ProgramsRepository, RunRulesOptions, ExportOptions } from "domain/repositories/ProgramsRepository"; import { D2Api, D2TrackedEntityInstanceToPost, @@ -59,11 +59,18 @@ export class ProgramsD2Repository implements ProgramsRepository { return programs; } - async export(options: { ids: Id[]; orgUnitIds: Id[] | undefined }): Async { - const { ids: programIds, orgUnitIds } = options; - const metadata = await this.getMetadata(programIds); - - const getOptions = { programIds, orgUnitIds }; + async export(options: ExportOptions): Async { + const { + ids: programIds, + orgUnitIds, + descendants: children, + skipMetadata, + updatedStartDate, + updatedEndDate, + } = options; + const metadata = skipMetadata ? undefined : await this.getMetadata(programIds); + + const getOptions = { programIds, orgUnitIds, children, updatedStartDate, updatedEndDate }; const events = await this.d2Tracker.getFromTracker("events", getOptions); const enrollments = await this.d2Tracker.getFromTracker("enrollments", getOptions); const trackedEntities = await this.d2Tracker.getFromTracker("trackedEntities", getOptions); @@ -75,7 +82,7 @@ export class ProgramsD2Repository implements ProgramsRepository { })); return { - metadata, + metadata: metadata, data: { events: events, enrollments: enrollments, @@ -114,8 +121,10 @@ export class ProgramsD2Repository implements ProgramsRepository { } async import(programExport: D2ProgramExport): Async { - const metadataRes = await runMetadata(this.api.metadata.post(programExport.metadata)); - log.info(`Metadata import status: ${metadataRes.status}`); + if (programExport.metadata) { + const metadataRes = await runMetadata(this.api.metadata.post(programExport.metadata)); + log.info(`Metadata import status: ${metadataRes.status}`); + } const { events, enrollments, trackedEntities } = programExport.data; const teisById = _.keyBy(trackedEntities, tei => tei.trackedEntity); @@ -139,7 +148,7 @@ export class ProgramsD2Repository implements ProgramsRepository { } interface D2ProgramExport { - metadata: object; + metadata?: object; data: D2ProgramData; } diff --git a/src/domain/entities/ProgramExport.ts b/src/domain/entities/ProgramExport.ts index 90c36eab..8c7dcb31 100644 --- a/src/domain/entities/ProgramExport.ts +++ b/src/domain/entities/ProgramExport.ts @@ -1,5 +1,5 @@ export interface ProgramExport { - metadata: object; + metadata?: object; data: ProgramData; } diff --git a/src/domain/repositories/ProgramsRepository.ts b/src/domain/repositories/ProgramsRepository.ts index f9556c33..64632e38 100644 --- a/src/domain/repositories/ProgramsRepository.ts +++ b/src/domain/repositories/ProgramsRepository.ts @@ -6,7 +6,7 @@ import { ProgramExport } from "domain/entities/ProgramExport"; export interface ProgramsRepository { get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async; - export(options: { ids: Id[] }): Async; + export(options: ExportOptions): Async; import(programExport: ProgramExport): Async; runRules(options: RunRulesOptions): Async; getAppUrl(options: { programId: Id; orgUnitId: Id }): string; @@ -25,3 +25,12 @@ export interface RunRulesOptions { payloadPath?: string; backup: boolean; } + +export interface ExportOptions { + ids: Id[]; + orgUnitIds?: Id[]; + updatedStartDate?: Timestamp; + updatedEndDate?: Timestamp; + descendants?: boolean; + skipMetadata?: boolean; +} diff --git a/src/domain/usecases/ExportProgramsUseCase.ts b/src/domain/usecases/ExportProgramsUseCase.ts index ae371c2d..f6ebdda3 100644 --- a/src/domain/usecases/ExportProgramsUseCase.ts +++ b/src/domain/usecases/ExportProgramsUseCase.ts @@ -1,6 +1,7 @@ import fs from "fs"; import { Async } from "domain/entities/Async"; import { Id } from "domain/entities/Base"; +import { Timestamp } from "domain/entities/Date"; import { ProgramsRepository } from "domain/repositories/ProgramsRepository"; import log from "utils/log"; @@ -20,4 +21,8 @@ interface ExportProgramsOptions { ids: Id[]; outputFile: string; orgUnitIds?: Id[]; + updatedStartDate?: Timestamp; + updatedEndDate?: Timestamp; + descendants?: boolean; + skipMetadata?: boolean; } diff --git a/src/scripts/commands/programs.ts b/src/scripts/commands/programs.ts index bf98a54a..93d90b83 100644 --- a/src/scripts/commands/programs.ts +++ b/src/scripts/commands/programs.ts @@ -9,6 +9,7 @@ import { getD2ApiFromArgs, StringPairSeparatedByDash, StringsSeparatedByCommas, + trackerUpdatedDate, } from "scripts/common"; import { ProgramsD2Repository } from "data/ProgramsD2Repository"; import { ExportProgramsUseCase } from "domain/usecases/ExportProgramsUseCase"; @@ -45,25 +46,50 @@ const programIdsOptions = option({ description: "List of program (comma-separated)", }); +const updatedStartDateArg = option({ + type: optional(trackerUpdatedDate("start")), + long: "updated-start-date", + description: + "Updated Start date: YYYY-MM-DD, YYYY-MM-DDTHH:mm or YYYY-MM-DDTHH:mm:ss.sss. A bare date defaults to the start of that day. Includes updatedAt equal or greater.", +}); + +const updatedEndDateArg = option({ + type: optional(trackerUpdatedDate("end")), + long: "updated-end-date", + description: + "Updated End date: YYYY-MM-DD, YYYY-MM-DDTHH:mm or YYYY-MM-DDTHH:mm:ss.sss. A bare date defaults to the end of that day. Includes updatedAt strictly lesser.", +}); + const exportCmd = command({ name: "export", description: "Export program metadata and data (events, enrollments, TEIs)", args: { - url: getApiUrlOption(), + ...getApiUrlOptions(), programIds: programIdsOptions, orgUnitIds: option({ type: optional(StringsSeparatedByCommas), long: "orgunits-ids", description: "List of organisation units (comma-separated)", }), + updatedStartDate: updatedStartDateArg, + updatedEndDate: updatedEndDateArg, + descendants: flag({ + long: "descendants", + description: "Include descendants of the specified organisation units", + }), + skipMetadata: flag({ + long: "skip-metadata", + description: "Skip metadata export", + }), outputFile: positional({ type: string, + displayName: "outputFile", description: "Output file (JSON)", }), }, handler: async args => { if (_.isEmpty(args.programIds)) throw new Error("Missing program IDs"); - const api = getD2Api(args.url); + const api = getD2ApiFromArgs(args); const programsRepository = new ProgramsD2Repository(api); new ExportProgramsUseCase(programsRepository).execute({ @@ -77,14 +103,14 @@ const importCmd = command({ name: "import", description: "Import program metadata and data (events, enrollments, TEIs)", args: { - url: getApiUrlOption(), + ...getApiUrlOptions(), inputFile: positional({ type: string, description: "Input file (JSON)", }), }, handler: async args => { - const api = getD2Api(args.url); + const api = getD2ApiFromArgs(args); const programsRepository = new ProgramsD2Repository(api); new ImportProgramsUseCase(programsRepository).execute({ inputFile: args.inputFile, diff --git a/src/scripts/common.ts b/src/scripts/common.ts index b8c732e1..4fbab10a 100644 --- a/src/scripts/common.ts +++ b/src/scripts/common.ts @@ -181,9 +181,38 @@ export const FilePath: Type = { }, }; +function isValidCalendarDateTime( + year: number, + month: number, + day: number, + hour: number, + minute: number, + second: number +): boolean { + const date = new Date(Date.UTC(year, month - 1, day, hour, minute, second)); + return ( + date.getUTCFullYear() === year && + date.getUTCMonth() === month - 1 && + date.getUTCDate() === day && + date.getUTCHours() === hour && + date.getUTCMinutes() === minute && + date.getUTCSeconds() === second + ); +} + function isValidDate(str: string): boolean { - const dateTimeRegex = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d{3})?)?$/; - return dateTimeRegex.test(str); + const match = str.match(/^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2}):(\d{2})(?:\.\d{3})?)?$/); + if (!match) return false; + + const [, year, month, day, hour = "0", minute = "0", second = "0"] = match; + return isValidCalendarDateTime( + Number(year), + Number(month), + Number(day), + Number(hour), + Number(minute), + Number(second) + ); } export const MetadataDate: Type = { @@ -197,3 +226,38 @@ export const MetadataDate: Type = { } }, }; + +export function trackerUpdatedDate(boundary: "start" | "end"): Type { + return { + async from(str) { + const match = str.match(/^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2})\.(\d{1,3}))?)?$/); + const invalidError = () => + new Error( + `Invalid date: ${str} (expected YYYY-MM-DD, YYYY-MM-DDTHH:mm or YYYY-MM-DDTHH:mm:ss.sss)` + ); + if (!match) throw invalidError(); + + const [, year, month, day, hour, minute, second, milliseconds] = match; + + if ( + !isValidCalendarDateTime( + Number(year), + Number(month), + Number(day), + Number(hour ?? "0"), + Number(minute ?? "0"), + Number(second ?? "0") + ) + ) { + throw invalidError(); + } + + if (hour === undefined) { + return `${year}-${month}-${day}T${boundary === "end" ? "23:59:59.999" : "00:00:00.000"}`; + } + return `${year}-${month}-${day}T${hour}:${minute}:${second ?? "00"}.${( + milliseconds ?? "000" + ).padEnd(3, "0")}`; + }, + }; +}