[Feature] Add start/end-date, descendants and skip-metadata to export programs script

README.md

@@ -109,6 +109,16 @@ Notes:
 
 Export a program with all its associated metadata and data (events, enrollments, tracked entities).
 
+The data can be filtered by:
+-   Organization Unit via the `--orgunits-ids` option. If `--descendants` is used, the included orgUnits will be the specified and its descendants. Note that if no `--orgunits-ids` is provided the `ouMode` will be `ALL` and the `--descendants` option will be redundant.
+-   The `lastUpdated` property using the `--updated-start-date` and/or `--updated-end-date` options. These options accept one of `YYYY-MM-DD`, `YYYY-MM-DDTHH:mm` or `YYYY-MM-DDTHH:mm:ss.sss`. A date (`YYYY-MM-DD`) defaults to the start of that day for `--updated-start-date` and the end of that day for `--updated-end-date` (e.g. `--updated-end-date=2026-01-01` is treated as `2026-01-01T23:59:59.999`). If a time is given, missing seconds/milliseconds default to zero, e.g. `2026-01-01T06:00` is treated as `2026-01-01T06:00:00.000` for both. These options apply the following criteria: `--updated-start-date` will include elements with `updatedAt` equal or greater, `--updated-end-date` will include elements with `updatedAt` strictly lesser than it.
+
+**Please note that the updated date options are applied independently to the three metadata types (events, enrollments, tracked entities). This can lead to the export referencing elements that are not in the export. For example events whose parent enrollment and/or TEI has an `updatedAt` date that falls outside of the date window**
+
+The metadata export can be skipped via the `--skip-metadata` option.
+
+The login details can be either provided as `--url='http://USER:PASSWORD@HOST:PORT'` or `--url='http://HOST:PORT' --auth='USER:PASSWORD'`. The `--auth` option is useful with complex passwords that can cause errors when used in the `--url` only approach.
+
 ```shell
 $ yarn start programs export --url='http://USER:PASSWORD@HOST:PORT' \
   --programs-ids=kX2GpLIa75l,kpNc7KvydVz programs.json

src/data/D2Tracker.ts

@@ -76,12 +76,14 @@ export class D2Tracker {
             orgUnitIds: string[] | undefined;
             trackedEntity?: string | undefined;
             children?: boolean;
+            updatedStartDate?: string;
+            updatedEndDate?: string;
         }
     ): Promise<Array<Mapping[Key][number]>> {
         type Output = Array<Mapping[Key][number]>;
 
         const output: Output = [];
-        const { programIds, orgUnitIds, trackedEntity } = options;
+        const { programIds, orgUnitIds, trackedEntity, updatedStartDate, updatedEndDate } = options;
 
         for (const programId of programIds) {
             let page = 1;
@@ -96,19 +98,22 @@ export class D2Tracker {
                 const apiOptions = {
                     page: page,
                     pageSize: pageSize,
+                    // NOTE: For 2.41+ ouMode is orgUnitMode
                     ouMode: ouMode as typeof ouMode,
                     orgUnit: orgUnitIds?.join(";"),
                     fields: { $all: true } as const,
                     program: programId,
                     trackedEntity,
+                    updatedAfter: updatedStartDate,
                 };
 
                 const { tracker } = this.api;
 
                 const endpoint = {
-                    trackedEntities: () => tracker.trackedEntities.get(apiOptions),
+                    trackedEntities: () =>
+                        tracker.trackedEntities.get({ ...apiOptions, updatedBefore: updatedEndDate }),
                     enrollments: () => tracker.enrollments.get(apiOptions),
-                    events: () => tracker.events.get(apiOptions),
+                    events: () => tracker.events.get({ ...apiOptions, updatedBefore: updatedEndDate }),
                 };
 
                 const res = await endpoint[model]().getData();
@@ -117,7 +122,7 @@ export class D2Tracker {
                 if (instances.length === 0) {
                     dataRemaining = false;
                 } else {
-                    output.push(...instances);
+                    output.push(...this.filterByUpdatedEndDate(model, instances, updatedEndDate));
                     page++;
                 }
             }
@@ -126,6 +131,21 @@ export class D2Tracker {
 
         return output;
     }
+
+    // NOTE: the enrollments endpoint has no updatedBefore param
+    private filterByUpdatedEndDate<Key extends TrackerDataKey>(
+        model: Key,
+        instances: Array<Mapping[Key][number]>,
+        updatedEndDate: string | undefined
+    ): Array<Mapping[Key][number]> {
+        if (!updatedEndDate || model !== "enrollments") return instances;
+
+        const endDate = new Date(updatedEndDate).getTime();
+
+        return (instances as Mapping["enrollments"]).filter(
+            enrollment => new Date(enrollment.updatedAt).getTime() < endDate
+        ) as Array<Mapping[Key][number]>;
+    }
 }
 
 type TrackerResponse = {

src/data/ProgramsD2Repository.ts

@@ -2,7 +2,7 @@ import _ from "lodash";
 import { Async } from "domain/entities/Async";
 import { Id } from "domain/entities/Base";
 import { ProgramExport } from "domain/entities/ProgramExport";
-import { ProgramsRepository, RunRulesOptions } from "domain/repositories/ProgramsRepository";
+import { ProgramsRepository, RunRulesOptions, ExportOptions } from "domain/repositories/ProgramsRepository";
 import {
     D2Api,
     D2TrackedEntityInstanceToPost,
@@ -59,11 +59,18 @@ export class ProgramsD2Repository implements ProgramsRepository {
         return programs;
     }
 
-    async export(options: { ids: Id[]; orgUnitIds: Id[] | undefined }): Async<ProgramExport> {
-        const { ids: programIds, orgUnitIds } = options;
-        const metadata = await this.getMetadata(programIds);
-
-        const getOptions = { programIds, orgUnitIds };
+    async export(options: ExportOptions): Async<ProgramExport> {
+        const {
+            ids: programIds,
+            orgUnitIds,
+            descendants: children,
+            skipMetadata,
+            updatedStartDate,
+            updatedEndDate,
+        } = options;
+        const metadata = skipMetadata ? undefined : await this.getMetadata(programIds);
+
+        const getOptions = { programIds, orgUnitIds, children, updatedStartDate, updatedEndDate };
         const events = await this.d2Tracker.getFromTracker("events", getOptions);
         const enrollments = await this.d2Tracker.getFromTracker("enrollments", getOptions);
         const trackedEntities = await this.d2Tracker.getFromTracker("trackedEntities", getOptions);
@@ -75,7 +82,7 @@ export class ProgramsD2Repository implements ProgramsRepository {
         }));
 
         return {
-            metadata,
+            metadata: metadata,
             data: {
                 events: events,
                 enrollments: enrollments,
@@ -114,8 +121,10 @@ export class ProgramsD2Repository implements ProgramsRepository {
     }
 
     async import(programExport: D2ProgramExport): Async<void> {
-        const metadataRes = await runMetadata(this.api.metadata.post(programExport.metadata));
-        log.info(`Metadata import status: ${metadataRes.status}`);
+        if (programExport.metadata) {
+            const metadataRes = await runMetadata(this.api.metadata.post(programExport.metadata));
+            log.info(`Metadata import status: ${metadataRes.status}`);
+        }
 
         const { events, enrollments, trackedEntities } = programExport.data;
         const teisById = _.keyBy(trackedEntities, tei => tei.trackedEntity);
@@ -139,7 +148,7 @@ export class ProgramsD2Repository implements ProgramsRepository {
 }
 
 interface D2ProgramExport {
-    metadata: object;
+    metadata?: object;
     data: D2ProgramData;
 }
 

src/domain/entities/ProgramExport.ts

@@ -1,5 +1,5 @@
 export interface ProgramExport {
-    metadata: object;
+    metadata?: object;
     data: ProgramData;
 }
 

src/domain/repositories/ProgramsRepository.ts

@@ -6,7 +6,7 @@ import { ProgramExport } from "domain/entities/ProgramExport";
 
 export interface ProgramsRepository {
     get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async<Program[]>;
-    export(options: { ids: Id[] }): Async<ProgramExport>;
+    export(options: ExportOptions): Async<ProgramExport>;
     import(programExport: ProgramExport): Async<void>;
     runRules(options: RunRulesOptions): Async<void>;
     getAppUrl(options: { programId: Id; orgUnitId: Id }): string;
@@ -25,3 +25,12 @@ export interface RunRulesOptions {
     payloadPath?: string;
     backup: boolean;
 }
+
+export interface ExportOptions {
+    ids: Id[];
+    orgUnitIds?: Id[];
+    updatedStartDate?: Timestamp;
+    updatedEndDate?: Timestamp;
+    descendants?: boolean;
+    skipMetadata?: boolean;
+}

src/domain/usecases/ExportProgramsUseCase.ts

@@ -1,6 +1,7 @@
 import fs from "fs";
 import { Async } from "domain/entities/Async";
 import { Id } from "domain/entities/Base";
+import { Timestamp } from "domain/entities/Date";
 import { ProgramsRepository } from "domain/repositories/ProgramsRepository";
 import log from "utils/log";
 
@@ -20,4 +21,8 @@ interface ExportProgramsOptions {
     ids: Id[];
     outputFile: string;
     orgUnitIds?: Id[];
+    updatedStartDate?: Timestamp;
+    updatedEndDate?: Timestamp;
+    descendants?: boolean;
+    skipMetadata?: boolean;
 }

src/scripts/commands/programs.ts

@@ -9,6 +9,7 @@ import {
     getD2ApiFromArgs,
     StringPairSeparatedByDash,
     StringsSeparatedByCommas,
+    trackerUpdatedDate,
 } from "scripts/common";
 import { ProgramsD2Repository } from "data/ProgramsD2Repository";
 import { ExportProgramsUseCase } from "domain/usecases/ExportProgramsUseCase";
@@ -45,25 +46,50 @@ const programIdsOptions = option({
     description: "List of program (comma-separated)",
 });
 
+const updatedStartDateArg = option({
+    type: optional(trackerUpdatedDate("start")),
+    long: "updated-start-date",
+    description:
+        "Updated Start date: YYYY-MM-DD, YYYY-MM-DDTHH:mm or YYYY-MM-DDTHH:mm:ss.sss. A bare date defaults to the start of that day. Includes updatedAt equal or greater.",
+});
+
+const updatedEndDateArg = option({
+    type: optional(trackerUpdatedDate("end")),
+    long: "updated-end-date",
+    description:
+        "Updated End date: YYYY-MM-DD, YYYY-MM-DDTHH:mm or YYYY-MM-DDTHH:mm:ss.sss. A bare date defaults to the end of that day. Includes updatedAt strictly lesser.",
+});
+
 const exportCmd = command({
     name: "export",
     description: "Export program metadata and data (events, enrollments, TEIs)",
     args: {
-        url: getApiUrlOption(),
+        ...getApiUrlOptions(),
         programIds: programIdsOptions,
         orgUnitIds: option({
             type: optional(StringsSeparatedByCommas),
             long: "orgunits-ids",
             description: "List of organisation units (comma-separated)",
         }),
+        updatedStartDate: updatedStartDateArg,
+        updatedEndDate: updatedEndDateArg,
+        descendants: flag({
+            long: "descendants",
+            description: "Include descendants of the specified organisation units",
+        }),
+        skipMetadata: flag({
+            long: "skip-metadata",
+            description: "Skip metadata export",
+        }),
         outputFile: positional({
             type: string,
+            displayName: "outputFile",
             description: "Output file (JSON)",
         }),
     },
     handler: async args => {
         if (_.isEmpty(args.programIds)) throw new Error("Missing program IDs");
-        const api = getD2Api(args.url);
+        const api = getD2ApiFromArgs(args);
         const programsRepository = new ProgramsD2Repository(api);
 
         new ExportProgramsUseCase(programsRepository).execute({
@@ -77,14 +103,14 @@ const importCmd = command({
     name: "import",
     description: "Import program metadata and data (events, enrollments, TEIs)",
     args: {
-        url: getApiUrlOption(),
+        ...getApiUrlOptions(),
         inputFile: positional({
             type: string,
             description: "Input file (JSON)",
         }),
     },
     handler: async args => {
-        const api = getD2Api(args.url);
+        const api = getD2ApiFromArgs(args);
         const programsRepository = new ProgramsD2Repository(api);
         new ImportProgramsUseCase(programsRepository).execute({
             inputFile: args.inputFile,

src/scripts/common.ts

@@ -181,9 +181,38 @@ export const FilePath: Type<string, string> = {
     },
 };
 
+function isValidCalendarDateTime(
+    year: number,
+    month: number,
+    day: number,
+    hour: number,
+    minute: number,
+    second: number
+): boolean {
+    const date = new Date(Date.UTC(year, month - 1, day, hour, minute, second));
+    return (
+        date.getUTCFullYear() === year &&
+        date.getUTCMonth() === month - 1 &&
+        date.getUTCDate() === day &&
+        date.getUTCHours() === hour &&
+        date.getUTCMinutes() === minute &&
+        date.getUTCSeconds() === second
+    );
+}
+
 function isValidDate(str: string): boolean {
-    const dateTimeRegex = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d{3})?)?$/;
-    return dateTimeRegex.test(str);
+    const match = str.match(/^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2}):(\d{2})(?:\.\d{3})?)?$/);
+    if (!match) return false;
+
+    const [, year, month, day, hour = "0", minute = "0", second = "0"] = match;
+    return isValidCalendarDateTime(
+        Number(year),
+        Number(month),
+        Number(day),
+        Number(hour),
+        Number(minute),
+        Number(second)
+    );
 }
 
 export const MetadataDate: Type<string, string> = {
@@ -197,3 +226,38 @@ export const MetadataDate: Type<string, string> = {
         }
     },
 };
+
+export function trackerUpdatedDate(boundary: "start" | "end"): Type<string, string> {
+    return {
+        async from(str) {
+            const match = str.match(/^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2})\.(\d{1,3}))?)?$/);
+            const invalidError = () =>
+                new Error(
+                    `Invalid date: ${str} (expected YYYY-MM-DD, YYYY-MM-DDTHH:mm or YYYY-MM-DDTHH:mm:ss.sss)`
+                );
+            if (!match) throw invalidError();
+
+            const [, year, month, day, hour, minute, second, milliseconds] = match;
+
+            if (
+                !isValidCalendarDateTime(
+                    Number(year),
+                    Number(month),
+                    Number(day),
+                    Number(hour ?? "0"),
+                    Number(minute ?? "0"),
+                    Number(second ?? "0")
+                )
+            ) {
+                throw invalidError();
+            }
+
+            if (hour === undefined) {
+                return `${year}-${month}-${day}T${boundary === "end" ? "23:59:59.999" : "00:00:00.000"}`;
+            }
+            return `${year}-${month}-${day}T${hour}:${minute}:${second ?? "00"}.${(
+                milliseconds ?? "000"
+            ).padEnd(3, "0")}`;
+        },
+    };
+}