diff --git a/src/data/ProgramsD2Repository.ts b/src/data/ProgramsD2Repository.ts index 45e47490..b5010e3e 100644 --- a/src/data/ProgramsD2Repository.ts +++ b/src/data/ProgramsD2Repository.ts @@ -8,6 +8,7 @@ import log from "utils/log"; import { promiseMap, runMetadata } from "./dhis2-utils"; import { D2ProgramRules } from "./d2-program-rules/D2ProgramRules"; import { D2Tracker } from "./D2Tracker"; +import { Program, ProgramType } from "domain/entities/Program"; type MetadataRes = { date: string } & { [k: string]: Array<{ id: string }> }; @@ -18,6 +19,39 @@ export class ProgramsD2Repository implements ProgramsRepository { this.d2Tracker = new D2Tracker(this.api); } + async get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async { + const { programs } = await this.api.metadata + .get({ + programs: { + fields: { + id: true, + name: true, + programType: true, + programStages: { + id: true, + name: true, + programStageDataElements: { + dataElement: { + id: true, + name: true, + code: true, + valueType: true, + optionSet: { id: true, name: true }, + }, + }, + }, + }, + filter: { + ...(options.ids ? { id: { in: options.ids } } : {}), + ...(options.programTypes ? { programType: { in: options.programTypes } } : {}), + }, + }, + }) + .getData(); + + return programs; + } + async export(options: { ids: Id[]; orgUnitIds: Id[] | undefined }): Async { const { ids: programIds, orgUnitIds } = options; const metadata = await this.getMetadata(programIds); diff --git a/src/domain/entities/Program.ts b/src/domain/entities/Program.ts new file mode 100644 index 00000000..e63e265b --- /dev/null +++ b/src/domain/entities/Program.ts @@ -0,0 +1,29 @@ +import { Maybe } from "utils/ts-utils"; +import { Id, NamedRef } from "./Base"; + +export type ProgramType = "WITH_REGISTRATION" | "WITHOUT_REGISTRATION"; + +export interface Program { + id: Id; + name: string; + programType: ProgramType; + programStages: ProgramStage[]; +} + +type ProgramStage = { + id: Id; + programStageDataElements: ProgramStageDataElement[]; +}; + +type ProgramStageDataElement = { + dataElement: DataElement; + displayInReports: boolean; +}; + +type DataElement = { + id: Id; + name: string; + code: string; + valueType: string; + optionSet: Maybe; +}; diff --git a/src/domain/entities/ProgramEvent.ts b/src/domain/entities/ProgramEvent.ts index 21082336..48b14fe4 100644 --- a/src/domain/entities/ProgramEvent.ts +++ b/src/domain/entities/ProgramEvent.ts @@ -21,7 +21,7 @@ export interface ProgramEventToSave { program: Ref; orgUnit: Ref; programStage: Ref; - dataValues: EventDataValue[]; + dataValues: EventDataValueToSave[]; trackedEntityInstanceId?: Id; status: EventStatus; date: Timestamp; @@ -43,6 +43,11 @@ export interface EventDataValue { lastUpdated: Timestamp; } +export interface EventDataValueToSave { + dataElementId: Id; + value: string; +} + export class DuplicatedProgramEvents { constructor( private options: { ignoreDataElementsIds: Maybe; checkDataElementsIds?: Maybe } diff --git a/src/domain/repositories/ProgramsRepository.ts b/src/domain/repositories/ProgramsRepository.ts index 3b05c68f..f91d4bc5 100644 --- a/src/domain/repositories/ProgramsRepository.ts +++ b/src/domain/repositories/ProgramsRepository.ts @@ -1,9 +1,11 @@ import { Async } from "domain/entities/Async"; import { Id } from "domain/entities/Base"; import { Timestamp } from "domain/entities/Date"; +import { Program, ProgramType } from "domain/entities/Program"; import { ProgramExport } from "domain/entities/ProgramExport"; export interface ProgramsRepository { + get(options: { ids?: Id[]; programTypes?: ProgramType[] }): Async; export(options: { ids: Id[] }): Async; import(programExport: ProgramExport): Async; runRules(options: RunRulesOptions): Async; diff --git a/src/domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase.ts b/src/domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase.ts new file mode 100644 index 00000000..17941a77 --- /dev/null +++ b/src/domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase.ts @@ -0,0 +1,188 @@ +import _ from "lodash"; +import { promiseMap } from "data/dhis2-utils"; +import { NamedRef } from "domain/entities/Base"; +import { D2Api } from "types/d2-api"; +import logger from "utils/log"; +import { D2TrackerTrackedEntity } from "@eyeseetea/d2-api/api/trackerTrackedEntities"; +import { D2TrackerEnrollment } from "@eyeseetea/d2-api/api/trackerEnrollments"; +import { D2TrackerEvent } from "@eyeseetea/d2-api/api/trackerEvents"; +import { ProgramsRepository } from "domain/repositories/ProgramsRepository"; + +export class DetectExternalOrgUnitUseCase { + pageSize = 1000; + + constructor(private api: D2Api, private programRepository: ProgramsRepository) {} + + async execute(options: { post: boolean }) { + const programs = await this.getPrograms(); + + await promiseMap(programs, async program => { + await this.fixEventsInProgram({ program: program, post: options.post }); + }); + } + + private async getPrograms() { + logger.info(`Get tracker programs`); + const programs = await this.programRepository.get({ programTypes: ["WITH_REGISTRATION"] }); + logger.info(`Total tracker programs: ${programs.length}`); + return programs; + } + + async fixEventsInProgram(options: { program: NamedRef; post: boolean }) { + const pageCount = await this.getPageCount(options); + + await promiseMap(_.range(1, pageCount + 1), async page => { + await this.fixEventsForPage({ ...options, page: page, pageCount: pageCount }); + }); + } + + private async getPageCount(options: { program: NamedRef; post: boolean }) { + const response = await this.api.tracker.trackedEntities + .get({ + ...params, + page: 1, + pageSize: 0, + totalPages: true, + program: options.program.id, + }) + .getData(); + + return Math.ceil((response.total || 0) / this.pageSize); + } + + async fixEventsForPage(options: { program: NamedRef; page: number; pageCount: number; post: boolean }) { + const trackedEntities = await this.getTrackedEntities(options); + const mismatchRecords = this.getMismatchRecords(trackedEntities); + const report = this.buildReport(mismatchRecords); + logger.info(`Events outside its enrollment orgUnit: ${mismatchRecords.length}`); + console.log(report); + + if (_(mismatchRecords).isEmpty()) { + logger.debug(`No events outside its enrollment orgUnit`); + } else if (!options.post) { + logger.info(`Add --post to update events (${mismatchRecords.length})`); + } else { + await this.fixMismatchEvents(mismatchRecords); + } + } + + private async fixMismatchEvents(mismatchRecords: MismatchRecord[]) { + const eventIds = mismatchRecords.map(obj => obj.event.event); + const mismatchRecordsByEventId = _.keyBy(mismatchRecords, obj => obj.event.event); + + logger.info(`Get events to update: ${eventIds.join(",")}`); + const { instances: events } = await this.api.tracker.events + .get({ fields: { $all: true }, event: eventIds.join(";") }) + .getData(); + + const fixedEvents = events.map((event): typeof event => { + const obj = mismatchRecordsByEventId[event.event]; + if (!obj) throw new Error(`Event not found: ${event.event}`); + return { ...event, orgUnit: obj.enrollment.orgUnit }; + }); + + await this.saveEvents(fixedEvents); + } + + private async saveEvents(fixedEvents: D2TrackerEvent[]) { + logger.info(`Post events: ${fixedEvents.length}`); + + const response = await this.api.tracker + .post( + { + async: false, + skipPatternValidation: true, + skipSideEffects: true, + skipRuleEngine: true, + importMode: "COMMIT", + }, + { events: fixedEvents } + ) + .getData(); + + logger.info(`Post result: ${JSON.stringify(response.stats)}`); + } + + private buildReport(mismatchRecords: MismatchRecord[]): string { + return mismatchRecords + .map(obj => { + const { trackedEntity: tei, enrollment: enr, event } = obj; + + const msg = [ + `trackedEntity: id=${tei.trackedEntity} orgUnit="${enr.orgUnitName}" [${enr.orgUnit}]`, + `event: id=${event.event} orgUnit="${event.orgUnitName}" [${event.orgUnit}]`, + ]; + + return msg.join(" - "); + }) + .join("\n"); + } + + private getMismatchRecords(trackedEntities: D2TrackerTrackedEntity[]): MismatchRecord[] { + return _(trackedEntities) + .flatMap(trackedEntity => { + return _(trackedEntity.enrollments) + .flatMap(enrollment => { + return enrollment.events.map(event => { + if (event.orgUnit !== enrollment.orgUnit) { + return { + trackedEntity: trackedEntity, + enrollment: enrollment, + event: event, + }; + } + }); + }) + .compact() + .value(); + }) + .value(); + } + + private async getTrackedEntities(options: { + program: NamedRef; + page: number; + post: boolean; + pageCount: number; + }): Promise { + logger.info(`Get events: page ${options.page} of ${options.pageCount}`); + + const response = await this.api.tracker.trackedEntities + .get({ + ...params, + page: options.page, + pageSize: this.pageSize, + program: options.program.id, + }) + .getData(); + + logger.info(`Tracked entities: ${response.instances.length}`); + + return response.instances; + } +} + +type MismatchRecord = { + trackedEntity: D2TrackerTrackedEntity; + enrollment: D2TrackerEnrollment; + event: D2TrackerEvent; +}; + +const params = { + ouMode: "ALL", + fields: { + trackedEntity: true, + orgUnit: true, + orgUnitName: true, + enrollments: { + enrollment: true, + orgUnit: true, + orgUnitName: true, + events: { + event: true, + orgUnit: true, + orgUnitName: true, + }, + }, + }, +} as const; diff --git a/src/domain/usecases/RecodeBooleanDataValuesInEventsUseCase.ts b/src/domain/usecases/RecodeBooleanDataValuesInEventsUseCase.ts new file mode 100644 index 00000000..6a876fdb --- /dev/null +++ b/src/domain/usecases/RecodeBooleanDataValuesInEventsUseCase.ts @@ -0,0 +1,228 @@ +import fs from "fs"; +import _ from "lodash"; +import { promiseMap } from "data/dhis2-utils"; +import { getId, Id, Ref } from "domain/entities/Base"; +import { D2Api } from "types/d2-api"; +import logger from "utils/log"; +import { D2TrackerEvent } from "@eyeseetea/d2-api/api/trackerEvents"; +import { ProgramEventsRepository } from "domain/repositories/ProgramEventsRepository"; +import { ProgramsRepository } from "domain/repositories/ProgramsRepository"; +import { Program } from "domain/entities/Program"; +import { Maybe } from "utils/ts-utils"; + +type Options = { + programId: Id; + ternaryOptionSetId: Id; + post: boolean; +}; + +export class RecodeBooleanDataValuesInEventsUseCase { + pageSize = 1000; + + constructor( + private api: D2Api, + private programsRepository: ProgramsRepository, + private eventsRepository: ProgramEventsRepository + ) {} + + async execute(options: Options) { + const program = await this.getProgram(options.programId); + await this.fixEventsInProgram({ ...options, program: program }); + } + + async getProgram(id: Id): Promise { + const programs = await this.programsRepository.get({ ids: [id] }); + const program = programs[0]; + if (!program) { + throw new Error(`Program not found: ${id}`); + } + return program; + } + + async fixEventsInProgram(options: { program: Program; post: boolean; ternaryOptionSetId: Id }) { + const pageCount = await this.getPageCount(options); + + await promiseMap(_.range(1, pageCount + 1), async page => { + await this.fixEventsForPage({ + ...options, + page: page, + pageCount: pageCount, + ternaryOptionSetId: options.ternaryOptionSetId, + }); + }); + } + + private async getPageCount(options: { program: Program }) { + const response = await this.api.tracker.events + .get({ + ...params, + page: 1, + pageSize: 0, + totalPages: true, + program: options.program.id, + }) + .getData(); + + const pageCount = Math.ceil((response.total || 0) / this.pageSize); + logger.info(`Total: ${response.total} -> pages: ${pageCount} (pageSize: ${this.pageSize})`); + + return pageCount; + } + + async fixEventsForPage(options: { + program: Program; + page: number; + pageCount: number; + post: boolean; + ternaryOptionSetId: Id; + }) { + const events = await this.getEvents(options); + const recodedEvents = this.getRecodedEvents({ + program: options.program, + events: events, + ternaryOptionSetId: options.ternaryOptionSetId, + }); + logger.info(`Events to recode: ${recodedEvents.length}`); + + if (_(recodedEvents).isEmpty()) { + return; + } else if (!options.post) { + logger.info(`Add --post to update events`); + } else { + await this.saveEvents(recodedEvents); + } + } + + getRecodedEvents(options: { + program: Program; + events: D2TrackerEvent[]; + ternaryOptionSetId: Id; + }): D2TrackerEvent[] { + const dataElementIdsWithTernary = this.getDataElementIdsWithTernaryOptionSet(options); + + return _(options.events) + .map((event): Maybe => { + return this.fixEvent(event, dataElementIdsWithTernary, options); + }) + .compact() + .value(); + } + + private fixEvent( + event: D2TrackerEvent, + dataElementIdsWithTernary: Set, + options: { program: Program } + ): Maybe { + const updatedDataValues = this.recodeEvent(event, dataElementIdsWithTernary); + if (_.isEqual(event.dataValues, updatedDataValues)) return undefined; + + const dataElementIdsUsedInDataValue = _(event.dataValues) + .map(dv => dv.dataElement) + .uniq() + .value(); + + const dataElementIdsInProgramStage = _(options.program.programStages) + .filter(programStage => programStage.id === event.programStage) + .flatMap(programStage => programStage.programStageDataElements) + .map(psde => psde.dataElement.id) + .uniq() + .value(); + + const dataElementIdsUsedAndNotCurrentlyAssigned = _.difference( + dataElementIdsUsedInDataValue, + dataElementIdsInProgramStage + ); + + if (!_(dataElementIdsUsedAndNotCurrentlyAssigned).isEmpty()) { + const tail = dataElementIdsUsedAndNotCurrentlyAssigned.join(" "); + const head = `[skip] event.id=${event.event}`; + const msg = `${head} [programStage.id=${event.programStage}] has unassigned dataElements: ${tail}`; + logger.error(msg); + return undefined; + } else { + return { ...event, dataValues: updatedDataValues }; + } + } + + private recodeEvent(event: D2TrackerEvent, dataElementIdsWithTernary: Set) { + return _(event.dataValues) + .map((dataValue): typeof dataValue => { + if (dataElementIdsWithTernary.has(dataValue.dataElement)) { + // TODO: User options from optionSet + const newValue = ["true", "Yes"].includes(dataValue.value) ? "Yes" : "No"; + return { ...dataValue, value: newValue }; + } else { + return dataValue; + } + }) + .value(); + } + + private getDataElementIdsWithTernaryOptionSet(options: { + program: Program; + events: D2TrackerEvent[]; + ternaryOptionSetId: Id; + }) { + const dataElements = _(options.program.programStages).flatMap(programStage => { + return programStage.programStageDataElements.map(programStageDataElement => { + return programStageDataElement.dataElement; + }); + }); + + const dataElementIdsWithTernaryOptions = new Set( + dataElements + .filter(dataElement => dataElement.optionSet?.id === options.ternaryOptionSetId) + .map(getId) + .value() + ); + return dataElementIdsWithTernaryOptions; + } + + private async saveEvents(events: D2TrackerEvent[]) { + logger.info(`Post events: ${events.length}`); + fs.writeFileSync("events.json", JSON.stringify(events, null, 2)); + + const response = await this.api.tracker + .post( + { + async: false, + skipPatternValidation: true, + skipSideEffects: true, + skipRuleEngine: true, + importMode: "COMMIT", + }, + { events: events } + ) + .getData(); + + logger.info(`Post result: ${JSON.stringify(response.stats)}`); + } + + private async getEvents(options: { + program: Ref; + page: number; + post: boolean; + pageCount: number; + }): Promise { + logger.info(`Get events: page ${options.page} of ${options.pageCount}`); + + const response = await this.api.tracker.events + .get({ + ...params, + page: options.page, + pageSize: this.pageSize, + program: options.program.id, + }) + .getData(); + + logger.info(`Events: ${response.instances.length}`); + + return response.instances; + } +} + +const params = { + fields: { + $all: true, + }, +} as const; diff --git a/src/scripts/commands/events.ts b/src/scripts/commands/events.ts index 15e5ef0f..f92755cc 100644 --- a/src/scripts/commands/events.ts +++ b/src/scripts/commands/events.ts @@ -1,12 +1,24 @@ import _ from "lodash"; import { command, string, subcommands, option, optional, flag } from "cmd-ts"; -import { getApiUrlOption, getD2Api, StringsSeparatedByCommas } from "scripts/common"; +import { + getApiUrlOption, + getApiUrlOptions, + getD2Api, + getD2ApiFromArgs, + StringsSeparatedByCommas, +} from "scripts/common"; import { ProgramEventsD2Repository } from "data/ProgramEventsD2Repository"; import { MoveEventsToOrgUnitUseCase } from "domain/usecases/MoveEventsToOrgUnitUseCase"; import logger from "utils/log"; import { UpdateEventDataValueUseCase } from "domain/usecases/UpdateEventDataValueUseCase"; import { EventExportSpreadsheetRepository } from "data/EventExportSpreadsheetRepository"; +import { D2Api } from "types/d2-api"; +import { Id } from "domain/entities/Base"; +import { promiseMap } from "data/dhis2-utils"; +import { DetectExternalOrgUnitUseCase } from "domain/usecases/ProcessEventsOutsideEnrollmentOrgUnitUseCase"; +import { ProgramsD2Repository } from "data/ProgramsD2Repository"; +import { RecodeBooleanDataValuesInEventsUseCase } from "domain/usecases/RecodeBooleanDataValuesInEventsUseCase"; export function getCommand() { return subcommands({ @@ -14,10 +26,30 @@ export function getCommand() { cmds: { "move-to-org-unit": moveOrgUnitCmd, "update-events": updateEventsDataValues, + "detect-external-orgunits": detectExternalOrgUnitCmd, + "recode-boolean-data-values": recodeBooleanDataValues, }, }); } +const detectExternalOrgUnitCmd = command({ + name: "detect-external-orgunits", + description: "Detect external organisation units", + args: { + ...getApiUrlOptions(), + post: flag({ + long: "post", + description: "Fix events", + defaultValue: () => false, + }), + }, + handler: async args => { + const api = getD2ApiFromArgs(args); + const programsRepository = new ProgramsD2Repository(api); + return new DetectExternalOrgUnitUseCase(api, programsRepository).execute(args); + }, +}); + const moveOrgUnitCmd = command({ name: "move-to-org-unit", description: "Move events to another organisation unit for event programs", @@ -114,3 +146,34 @@ const updateEventsDataValues = command({ } }, }); + +const recodeBooleanDataValues = command({ + name: "recode-boolean-data-values", + description: "Recode boolean data values", + args: { + ...getApiUrlOptions(), + programId: option({ + type: string, + long: "program-id", + description: "Program ID to recode", + }), + ternaryOptionSetId: option({ + type: string, + long: "ternary-optionset-id", + description: "ID of the ternary option set (Yes/No/NA) to recode", + }), + post: flag({ + long: "post", + description: "Fix events", + defaultValue: () => false, + }), + }, + handler: async args => { + const api = getD2ApiFromArgs(args); + const eventsRepository = new ProgramEventsD2Repository(api); + const programsRepository = new ProgramsD2Repository(api); + return new RecodeBooleanDataValuesInEventsUseCase(api, programsRepository, eventsRepository).execute( + args + ); + }, +}); diff --git a/src/utils/log.ts b/src/utils/log.ts index 35afd7f8..5ff51ae8 100644 --- a/src/utils/log.ts +++ b/src/utils/log.ts @@ -3,6 +3,7 @@ import { isElementOfUnion, UnionFromValues } from "./ts-utils"; const logLevels = ["debug", "info", "warn", "error"] as const; export type LogLevel = UnionFromValues; +// data layer -> domain layer const levelFromEnv = process.env["LOG_LEVEL"] || ""; const level = isElementOfUnion(levelFromEnv, logLevels) ? levelFromEnv : "info"; const levelIndex = logLevels.indexOf(level);