Stats + adatpt OutsideEnrollment for new tracker (clean arch)

src/data/CategoryOptionD2Repository.ts

@@ -74,26 +74,28 @@ export class CategoryOptionD2Repository implements CategoryOptionRepository {
                 {
                     recordsSkipped: saveResponse.status === "ERROR" ? catOptionsToSave.map(co => co.id) : [],
                     errorMessage,
-                    created: saveResponse.response.stats.created,
-                    ignored: saveResponse.response.stats.ignored,
-                    updated: saveResponse.response.stats.updated,
+                    ...saveResponse.response.stats,
                 },
             ];
         });
 
         return stats.reduce(
-            (acum, stat) => {
+            (acum, stat): Stats => {
                 return {
                     recordsSkipped: [...acum.recordsSkipped, ...stat.recordsSkipped],
                     errorMessage: `${acum.errorMessage}${stat.errorMessage}`,
                     created: acum.created + stat.created,
                     ignored: acum.ignored + stat.ignored,
                     updated: acum.updated + stat.updated,
+                    deleted: acum.deleted + stat.deleted,
+                    total: acum.total + stat.total,
                 };
             },
             {
                 recordsSkipped: [],
                 errorMessage: "",
+                deleted: 0,
+                total: 0,
                 created: 0,
                 ignored: 0,
                 updated: 0,

src/data/D2Tracker.ts

@@ -5,11 +5,18 @@ import { Async } from "domain/entities/Async";
 import { Stats } from "domain/entities/Stats";
 import { D2Api } from "types/d2-api";
 import log from "utils/log";
+import { TrackerPostRequest } from "@eyeseetea/d2-api/api/tracker";
+import { TrackedEntitiesGetResponse } from "@eyeseetea/d2-api/api/trackerTrackedEntities";
+import { TrackerEnrollmentsResponse } from "@eyeseetea/d2-api/api/trackerEnrollments";
+import { TrackerEventsResponse } from "@eyeseetea/d2-api/api/trackerEvents";
 
 export class D2Tracker {
     constructor(private api: D2Api) {}
 
-    async postTracker(key: TrackerDataKey, objects: object[]): Async<TrackerResponse[]> {
+    async postTracker<Key extends TrackerDataKey>(
+        key: Key,
+        objects: Array<NonNullable<TrackerPostRequest[Key]>[number]>
+    ): Async<TrackerResponse[]> {
         const total = objects.length;
         log.info(`Import data: ${key} - Total: ${total}`);
         let page = 1;
@@ -29,17 +36,21 @@ export class D2Tracker {
         return result;
     }
 
-    private async postTrackerData(data: object, options: { payloadId: string }): Async<TrackerResponse> {
-        const response: TrackerResponse = await this.api
-            .post<TrackerResponse>("/tracker", { async: false }, data)
+    private async postTrackerData(
+        data: TrackerPostRequest,
+        options: { payloadId: string }
+    ): Async<TrackerResponse> {
+        const response: TrackerResponse = await this.api.tracker
+            .post({ async: false }, data)
             .getData()
-            .catch(err => {
-                if (err?.response?.data) {
-                    return err.response.data as TrackerResponse;
+            .then(res => ({ ...res, stats: { ...Stats.empty(), ...res.stats } }))
+            .catch((err): TrackerResponse => {
+                const data = err?.response?.data;
+                if (data) {
+                    return data;
                 } else {
                     return {
                         status: "ERROR",
-                        typeReports: [],
                         stats: Stats.empty(),
                     };
                 }
@@ -56,37 +67,47 @@ export class D2Tracker {
         }
     }
 
-    async getFromTracker<T>(
-        apiPath: string,
+    async getFromTracker<Key extends TrackerDataKey>(
+        model: Key,
         options: {
             programIds: string[];
             orgUnitIds: string[] | undefined;
-            fields?: string;
             trackedEntity?: string | undefined;
         }
-    ): Promise<T[]> {
-        const output = [];
-        const { programIds, orgUnitIds, fields = "*", trackedEntity } = options;
+    ): Promise<Array<Mapping[Key][number]>> {
+        type Output = Array<Mapping[Key][number]>;
+
+        const output: Output = [];
+        const { programIds, orgUnitIds, trackedEntity } = options;
 
         for (const programId of programIds) {
             let page = 1;
             let dataRemaining = true;
 
             while (dataRemaining) {
                 const pageSize = 1000;
-                log.debug(`GET ${apiPath} (pageSize=${pageSize}, page=${page})`);
-
-                const { instances } = await this.api
-                    .get<{ instances: T[] }>(`/tracker/${apiPath}`, {
-                        page,
-                        pageSize: pageSize,
-                        ouMode: orgUnitIds ? "SELECTED" : "ALL",
-                        orgUnit: orgUnitIds?.join(";"),
-                        fields: fields,
-                        program: programId,
-                        trackedEntity,
-                    })
-                    .getData();
+                log.debug(`GET ${model} (pageSize=${pageSize}, page=${page})`);
+
+                const apiOptions = {
+                    page,
+                    pageSize: pageSize,
+                    ouMode: orgUnitIds ? ("SELECTED" as const) : ("ALL" as const),
+                    orgUnit: orgUnitIds?.join(";"),
+                    fields: { $all: true as const },
+                    program: programId,
+                    trackedEntity,
+                };
+
+                const { tracker } = this.api;
+
+                const endpoint = {
+                    trackedEntities: () => tracker.trackedEntities.get(apiOptions),
+                    enrollments: () => tracker.enrollments.get(apiOptions),
+                    events: () => tracker.events.get(apiOptions),
+                };
+
+                const res = await endpoint[model]().getData();
+                const instances: Output = res.instances as Output;
 
                 if (instances.length === 0) {
                     dataRemaining = false;
@@ -96,16 +117,21 @@ export class D2Tracker {
                 }
             }
         }
-        log.info(`GET ${apiPath} -> Total: ${output.length}`);
+        log.info(`GET ${model} -> Total: ${output.length}`);
 
         return output;
     }
 }
 
 type TrackerResponse = {
     status: string;
-    typeReports: object[];
     stats: Stats;
 };
 
 type TrackerDataKey = "events" | "enrollments" | "trackedEntities";
+
+type Mapping = {
+    trackedEntities: TrackedEntitiesGetResponse<{ $all: true }>["instances"];
+    enrollments: TrackerEnrollmentsResponse<{ $all: true }>["instances"];
+    events: TrackerEventsResponse<{ $all: true }>["instances"];
+};

src/data/ProgramEventsD2Repository.ts

@@ -5,7 +5,7 @@ import { GetOptions, ProgramEventsRepository } from "domain/repositories/Program
 import { D2Api, Ref } from "types/d2-api";
 import { cartesianProduct } from "utils/array";
 import logger from "utils/log";
-import { getId, Id } from "domain/entities/Base";
+import { getId, Id, NamedRef } from "domain/entities/Base";
 import { Result } from "domain/entities/Result";
 import { getInChunks } from "./dhis2-utils";
 import { promiseMap } from "./dhis2-utils";
@@ -41,48 +41,10 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository {
     constructor(private api: D2Api) {}
 
     async get(options: GetOptions): Async<ProgramEvent[]> {
+        const d2EventsMapper = await D2EventsMapper.build(this.api);
         const d2Events = await this.getD2Events(options);
 
-        const { programs } = await this.api.metadata
-            .get({
-                programs: {
-                    fields: {
-                        id: true,
-                        name: true,
-                        programStages: { id: true, name: true },
-                    },
-                },
-            })
-            .getData();
-
-        const programsById = _.keyBy(programs, getId);
-
-        const programStagesById = _(programs)
-            .flatMap(program => program.programStages)
-            .uniqBy(getId)
-            .keyBy(getId)
-            .value();
-
-        return d2Events.map(
-            (event): ProgramEvent => ({
-                created: event.createdAt,
-                id: event.event,
-                program: programsById[event.program] || { id: event.program, name: "" },
-                programStage: programStagesById[event.programStage] || { id: event.programStage, name: "" },
-                orgUnit: { id: event.orgUnit, name: event.orgUnitName },
-                trackedEntityInstanceId: event.trackedEntity,
-                status: event.status,
-                date: event.occurredAt,
-                dueDate: event.scheduledAt,
-                dataValues: event.dataValues.map(dv => ({
-                    dataElementId: dv.dataElement,
-                    value: dv.value,
-                    storedBy: dv.storedBy,
-                    providedElsewhere: dv.providedElsewhere,
-                    lastUpdated: dv.updatedAt,
-                })),
-            })
-        );
+        return d2Events.map(d2Event => d2EventsMapper.getEventEntityFromD2Object(d2Event));
     }
 
     async delete(events: Ref[]): Async<Result> {
@@ -205,13 +167,77 @@ export class ProgramEventsD2Repository implements ProgramEventsRepository {
     }
 }
 
+export class D2EventsMapper {
+    constructor(
+        private programsById: Record<Id, NamedRef>,
+        private programStagesById: Record<Id, NamedRef>
+    ) {}
+
+    static async build(api: D2Api) {
+        const { programs } = await api.metadata
+            .get({
+                programs: {
+                    fields: {
+                        id: true,
+                        name: true,
+                        programStages: { id: true, name: true },
+                    },
+                },
+            })
+            .getData();
+
+        const programsById = _.keyBy(programs, getId);
+
+        const programStagesById = _(programs)
+            .flatMap(program => program.programStages)
+            .uniqBy(getId)
+            .keyBy(getId)
+            .value();
+
+        return new D2EventsMapper(programsById, programStagesById);
+    }
+
+    getEventEntityFromD2Object(event: Event): ProgramEvent {
+        return {
+            id: event.event,
+            created: event.createdAt,
+            program: this.programsById[event.program] || { id: event.program, name: "" },
+            programStage: this.programStagesById[event.programStage] || { id: event.programStage, name: "" },
+            orgUnit: { id: event.orgUnit, name: event.orgUnitName },
+            trackedEntityInstanceId: event.trackedEntity,
+            status: event.status,
+            date: event.occurredAt,
+            dueDate: event.scheduledAt,
+            dataValues: event.dataValues.map(dv => ({
+                dataElementId: dv.dataElement,
+                value: dv.value,
+                storedBy: dv.storedBy,
+                providedElsewhere: dv.providedElsewhere,
+                lastUpdated: dv.updatedAt,
+            })),
+        };
+    }
+}
+
 type EventToPost = NonNullable<TrackerPostRequest["events"]>[number];
 
 async function importEvents(api: D2Api, events: EventToPost[], params?: TrackerPostParams): Async<Result> {
     if (_.isEmpty(events)) return { type: "success", message: "No events to post" };
 
     const resList = await promiseMap(_.chunk(events, 100), async eventsGroup => {
-        const res = await api.tracker.post(params || {}, { events: eventsGroup }).getData();
+        const res = await api.tracker
+            .post(
+                {
+                    async: false,
+                    skipPatternValidation: true,
+                    skipSideEffects: true,
+                    skipRuleEngine: true,
+                    importMode: "COMMIT",
+                    ...params,
+                },
+                { events: eventsGroup }
+            )
+            .getData();
         if (res.status === "OK") {
             const message = JSON.stringify(
                 _.pick(res, ["status", "imported", "updated", "deleted", "ignored"])

src/data/ProgramsD2Repository.ts

@@ -9,6 +9,9 @@ import { promiseMap, runMetadata } from "./dhis2-utils";
 import { D2ProgramRules } from "./d2-program-rules/D2ProgramRules";
 import { D2Tracker } from "./D2Tracker";
 import { Program, ProgramType } from "domain/entities/Program";
+import { D2TrackerEnrollmentToPost } from "@eyeseetea/d2-api/api/trackerEnrollments";
+import { D2TrackerEventToPost } from "@eyeseetea/d2-api/api/trackerEvents";
+import { D2TrackedEntityInstanceToPost } from "@eyeseetea/d2-api/api/trackerTrackedEntities";
 
 type MetadataRes = { date: string } & { [k: string]: Array<{ id: string }> };
 
@@ -43,12 +46,9 @@ export class ProgramsD2Repository implements ProgramsRepository {
         const metadata = await this.getMetadata(programIds);
 
         const getOptions = { programIds, orgUnitIds };
-        const events = await this.d2Tracker.getFromTracker<object>("events", getOptions);
-        const enrollments = await this.d2Tracker.getFromTracker<D2Enrollment>("enrollments", getOptions);
-        const trackedEntities = await this.d2Tracker.getFromTracker<D2TrackedEntity>(
-            "trackedEntities",
-            getOptions
-        );
+        const events = await this.d2Tracker.getFromTracker("events", getOptions);
+        const enrollments = await this.d2Tracker.getFromTracker("enrollments", getOptions);
+        const trackedEntities = await this.d2Tracker.getFromTracker("trackedEntities", getOptions);
 
         /* Remove redundant enrollments info from TEIs */
         const trackedEntitiesWithoutEnrollments = trackedEntities.map(trackedEntity => ({
@@ -59,7 +59,7 @@ export class ProgramsD2Repository implements ProgramsRepository {
         return {
             metadata,
             data: {
-                events,
+                events: events,
                 enrollments: enrollments,
                 trackedEntities: trackedEntitiesWithoutEnrollments,
             },
@@ -98,7 +98,7 @@ export class ProgramsD2Repository implements ProgramsRepository {
         // DHIS2 exports enrollments without attributes, but requires it on import, add from TEI
         const enrollmentsWithAttributes = enrollments.map(enrollment => ({
             ...enrollment,
-            attributes: teisById[enrollment.trackedEntity]?.attributes || [],
+            attributes: (enrollment.trackedEntity && teisById[enrollment.trackedEntity]?.attributes) || [],
         }));
 
         log.info(`Import data`);
@@ -119,16 +119,11 @@ interface D2ProgramExport {
 }
 
 type D2ProgramData = {
-    events: object[];
-    enrollments: D2Enrollment[];
-    trackedEntities: D2TrackedEntity[];
+    events: D2TrackerEventToPost[];
+    enrollments: D2TrackerEnrollmentToPost[];
+    trackedEntities: D2TrackedEntityInstanceToPost[];
 };
 
-interface D2Enrollment {
-    enrollment: string;
-    trackedEntity: string;
-}
-
 export interface D2TrackedEntity {
     trackedEntity: Id;
     orgUnit: Id;