Merge pull request #928 from EyeSeeTea/development

Release 2.16.0

README.md

@@ -20,10 +20,13 @@ $ yarn migrate 'http://admin:PASSWORD@localhost:8080'
 
 ## Scheduler
 
-The app provides a server-side scheduler script that runs synchronization rules in the background. The script requires Node v10+ and can be executed like this:
+The app provides a server-side scheduler script that runs synchronization rules in the background. The script requires Node v10+.
+
+* Unzip metadata-synchronization-server.zip and can be executed like this:
 
 ```
-$ node metadata-synchronization-server.js -c app-config.json
+$ cd metadata-synchronization-server
+$ node index.js -c app-config.json
 ```
 
 To connect to the destination instance, it requires a configuration file. If no configuration file is supplied the following is used as a placeholder:
@@ -121,6 +124,8 @@ To build the scheduler:
 $ yarn build-scheduler
 ```
 
+This script generate a metadata-synchronization-server.zip file.
+
 ## i18n
 
 ### Update an existing language

i18n/en.pot

@@ -5,8 +5,8 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n != 1)\n"
-"POT-Creation-Date: 2021-12-30T06:26:16.031Z\n"
-"PO-Revision-Date: 2021-12-30T06:26:16.031Z\n"
+"POT-Creation-Date: 2022-04-21T07:57:05.960Z\n"
+"PO-Revision-Date: 2022-04-21T07:57:05.960Z\n"
 
 msgid ""
 "THIS NEW RELEASE INCLUDES SHARING SETTINGS PER INSTANCES. FOR THIS VERSION "

package.json

@@ -1,7 +1,7 @@
 {
     "name": "metadata-synchronization",
     "description": "Advanced metadata & data synchronization utility",
-    "version": "2.15.0",
+    "version": "2.16.0",
     "license": "GPL-3.0",
     "author": "EyeSeeTea team",
     "homepage": ".",
@@ -65,7 +65,7 @@
         "craco-start": "craco start",
         "start-scheduler": "yarn run-ts --files src/scheduler/cli.ts",
         "build": "yarn run-ts scripts/run.ts build",
-        "build-scheduler": "ncc build src/scheduler/cli.ts -m && cp dist/index.js $npm_package_name-server.js",
+        "build-scheduler": "ncc build src/scheduler/cli.ts -m -o $npm_package_name-server && zip -r $npm_package_name-server.zip $npm_package_name-server && npx rimraf $npm_package_name-server/",
         "run-ts": "ts-node --files -O '{\"module\":\"commonjs\"}'",
         "migrate": "yarn run-ts src/migrations/cli.ts",
         "test": "jest",
@@ -104,7 +104,7 @@
         "@typescript-eslint/eslint-plugin": "5.0.0",
         "@typescript-eslint/parser": "5.0.0",
         "@welldone-software/why-did-you-render": "6.2.1",
-        "@zeit/ncc": "0.22.3",
+        "@vercel/ncc": "0.33.3",
         "babel-core": "6.26.3",
         "babel-eslint": "10.1.0",
         "craco": "0.0.3",

src/data/events/EventsD2ApiRepository.ts

@@ -183,32 +183,48 @@ export class EventsD2ApiRepository implements EventsRepository {
 
     public async save(data: EventsPackage, params: DataImportParams = {}): Promise<SynchronizationResult> {
         try {
-            const { response } = await this.api.events
-                .postAsync(
-                    {
-                        idScheme: params.idScheme ?? "UID",
-                        dataElementIdScheme: params.dataElementIdScheme ?? "UID",
-                        orgUnitIdScheme: params.orgUnitIdScheme ?? "UID",
-                        dryRun: params.dryRun ?? false,
-                        preheatCache: params.preheatCache ?? false,
-                        skipExistingCheck: params.skipExistingCheck ?? false,
-                    },
-                    data
-                )
-                .getData();
-
-            const result = await this.api.system.waitFor(response.jobType, response.id).getData();
-
-            if (!result) {
+            if (data.events.length === 0) {
                 return {
-                    status: "ERROR",
+                    status: "SUCCESS",
+                    stats: {
+                        imported: 0,
+                        updated: 0,
+                        deleted: 0,
+                        ignored: 0,
+                        total: 0,
+                    },
                     instance: this.instance.toPublicObject(),
                     date: new Date(),
                     type: "events",
                 };
-            }
+            } else {
+                const { response } = await this.api.events
+                    .postAsync(
+                        {
+                            idScheme: params.idScheme ?? "UID",
+                            dataElementIdScheme: params.dataElementIdScheme ?? "UID",
+                            orgUnitIdScheme: params.orgUnitIdScheme ?? "UID",
+                            dryRun: params.dryRun ?? false,
+                            preheatCache: params.preheatCache ?? false,
+                            skipExistingCheck: params.skipExistingCheck ?? false,
+                        },
+                        data
+                    )
+                    .getData();
+
+                const result = await this.api.system.waitFor(response.jobType, response.id).getData();
 
-            return this.cleanEventsImportResponse(result);
+                if (!result) {
+                    return {
+                        status: "ERROR",
+                        instance: this.instance.toPublicObject(),
+                        date: new Date(),
+                        type: "events",
+                    };
+                }
+
+                return this.cleanEventsImportResponse(result);
+            }
         } catch (error: any) {
             if (error?.response?.data) {
                 return this.cleanEventsImportResponse(error.response.data);

src/data/instance/InstanceD2ApiRepository.ts

@@ -193,7 +193,8 @@ export class InstanceD2ApiRepository implements InstanceRepository {
             ...data,
             url: data.type === "local" ? this.instance.url : data.url,
             version: data.type === "local" ? this.instance.version : data.version,
-            password: this.decryptPassword(data.password),
+            username: data.type === "local" ? this.instance.username : data.username,
+            password: data.type === "local" ? this.instance.password : this.decryptPassword(data.password),
             ...sharing,
         });
     }

src/data/migrations/tasks/08.remove-coc-inner-mappings.ts

@@ -14,6 +14,28 @@ interface InstanceDetails {
     password?: string;
 }
 
+async function getMetadata(d2Api: D2Api, ids: string[]): Promise<MetadataPackage> {
+    const promises = [];
+    const chunkSize = 50;
+
+    for (let i = 0; i < ids.length; i += chunkSize) {
+        const requestIds = ids.slice(i, i + chunkSize).toString();
+
+        promises.push(
+            d2Api
+                .get<MetadataPackage>("/metadata", {
+                    fields: "id",
+                    filter: "id:in:[" + requestIds + "]",
+                })
+                .getData()
+        );
+    }
+    const response = await Promise.all(promises);
+    const results = _.deepMerge({}, ...response);
+    if (results.system) delete results.system;
+    return results;
+}
+
 function cleanProgramDataElements(oldProgramDataElements: { [id: string]: MetadataMapping }) {
     return oldProgramDataElements
         ? Object.keys(oldProgramDataElements).reduce((previous, key) => {
@@ -35,12 +57,8 @@ function cleanProgramDataElements(oldProgramDataElements: { [id: string]: Metada
 async function cleanAggregatedDataElements(d2Api: D2Api, oldAggregatedItems: { [id: string]: MetadataMapping }) {
     // aggregatedDataElements mappping key can contain dataElements, indicators and programIndicators
     // We only need remove categoryOptionCombos from the inner mapping for data elements
-    const aggregatedMetadata = await d2Api
-        .get<MetadataPackage>("/metadata", {
-            fields: "id",
-            filter: "id:in:[" + Object.keys(oldAggregatedItems) + "]",
-        })
-        .getData();
+    const oldAggregatedDataElements = Object.keys(oldAggregatedItems);
+    const aggregatedMetadata = await getMetadata(d2Api, oldAggregatedDataElements);
 
     return oldAggregatedItems
         ? Object.keys(oldAggregatedItems).reduce((previous, key) => {

src/data/transformations/PackageTransformations.ts

@@ -271,13 +271,13 @@ export const metadataTransformations: Transformation[] = [
 
                 const newCharts = charts.map((chart: any) => {
                     return {
-                        series: (chart.columnDimensions || [])[0],
                         category: (chart.rowDimensions || [])[0],
                         seriesItems: (chart.optionalAxes || []).map(({ dimensionalItem, axis }: any) => ({
                             series: dimensionalItem,
                             axis,
                         })),
                         ...chart,
+                        series: (chart.columnDimensions || [])[0],
                     };
                 });
 
@@ -403,6 +403,31 @@ export const metadataTransformations: Transformation[] = [
             }
         },
     },
+    {
+        name: "fix duplicate translations error",
+        apiVersion: 36,
+        apply: (metadata: any) => {
+            const fixedMetadata = _.omitBy(metadata, _.isNil);
+
+            const newMetadata = Object.keys(fixedMetadata).reduce((acc, key) => {
+                const items = metadata[key].map((item: any) => {
+                    return {
+                        ...item,
+                        translations: item.translations
+                            ? _.uniqWith(
+                                  item.translations,
+                                  (transA: any, transB: any) =>
+                                      transA.property === transB.property && transA.locale === transB.locale
+                              )
+                            : item.translations,
+                    };
+                });
+                return { ...acc, [key]: items };
+            }, {});
+
+            return newMetadata;
+        },
+    },
 ];
 
 const itemsMapping = {
@@ -488,7 +513,7 @@ function getPeriodDataFromYearlySeries(object: any) {
                 .fromPairs()
                 .value(),
         },
-        periods: years.map(year => ({ id: year })),
+        periods: object.periods?.length > 0 ? object.periods : years.map(year => ({ id: year })),
     };
 
     return periodsData;

src/data/transformations/__tests__/integration/transformations-api-36.spec.ts

@@ -0,0 +1,66 @@
+import { sync } from "./helpers";
+
+describe("API 36", () => {
+    beforeAll(() => {
+        jest.setTimeout(30000);
+    });
+
+    describe("Transformation 2.34 -> 2.36", () => {
+        it("Remove duplicate transformations", async () => {
+            const metadata = {
+                dataElements: [
+                    {
+                        code: "BU_EPI_Gender",
+                        lastUpdated: "2020-10-30T16:12:08.461",
+                        id: "zqv3IsyTYta",
+                        created: "2018-05-09T22:26:29.218",
+                        name: "BU_EPI_Gender",
+                        shortName: "# BU cases (by gender)",
+                        aggregationType: "SUM",
+                        domainType: "AGGREGATE",
+                        displayName: "BU_EPI_Gender",
+                        translations: [
+                            {
+                                property: "SHORT_NAME",
+                                locale: "fr",
+                                value: "# cas d'UB par sexe",
+                            },
+                            {
+                                property: "NAME",
+                                locale: "fr",
+                                value: "Nombre de cas d'ulc�re de Buruli (par genre)",
+                            },
+                            {
+                                property: "SHORT_NAME",
+                                locale: "fr",
+                                value: "Cas d'ulc�re de Buruli (par genre)",
+                            },
+                            {
+                                property: "FORM_NAME",
+                                locale: "fr",
+                                value: "Nombre de nouveaux cas d'UB par sexe",
+                            },
+                        ],
+                    },
+                ],
+            };
+
+            const { dataElements } = await sync({
+                from: "2.34",
+                to: "2.36",
+                metadata,
+                models: ["dataElements"],
+            });
+
+            const translations = dataElements["zqv3IsyTYta"].translations as any[];
+
+            const translationsOldDuplicate = translations.filter(
+                trans => trans.property === "SHORT_NAME" && trans.locale === "fr"
+            );
+
+            expect(translationsOldDuplicate.length).toBe(1);
+        });
+    });
+});
+
+export {};

src/presentation/webapp/msf-aggregate-data/pages/MSFHomePagePresenter.ts

@@ -4,7 +4,6 @@ import { DataSyncAggregation } from "../../../../domain/aggregated/entities/Data
 import { DataSyncPeriod } from "../../../../domain/aggregated/entities/DataSyncPeriod";
 import { buildPeriodFromParams } from "../../../../domain/aggregated/utils";
 import { Instance } from "../../../../domain/instance/entities/Instance";
-import { ProgramIndicator } from "../../../../domain/metadata/entities/MetadataEntities";
 import { SynchronizationReport } from "../../../../domain/reports/entities/SynchronizationReport";
 import { SynchronizationRule } from "../../../../domain/rules/entities/SynchronizationRule";
 import { Store } from "../../../../domain/stores/entities/Store";
@@ -18,7 +17,7 @@ import { formatDateLong } from "../../../../utils/date";
 import { availablePeriods } from "../../../../utils/synchronization";
 import { CompositionRoot } from "../../../CompositionRoot";
 import { AdvancedSettings, MSFSettings } from "./MSFEntities";
-import { NamedRef } from "../../../../domain/common/entities/Ref";
+import { NamedRef, Ref } from "../../../../domain/common/entities/Ref";
 
 type LoggerFunction = (event: string, userType?: "user" | "admin") => void;
 
@@ -140,7 +139,7 @@ async function validatePreviousDataValues(
             if (!rule.dataParams || !rule.dataParams.period) return undefined;
             const { startDate } = buildPeriodFromParams(rule.dataParams);
 
-            const programs = await getRulePrograms(compositionRoot, rule, instance);
+            const programStageIds = await getRuleProgramStageIds(compositionRoot, rule, instance);
 
             const events = await promiseMap(rule.dataSyncOrgUnitPaths, async orgUnit => {
                 const executionKey = `${rule.id}-${cleanOrgUnitPath(orgUnit)}`;
@@ -156,7 +155,7 @@ async function validatePreviousDataValues(
                         orgUnitPaths: [orgUnit],
                         allEvents: true,
                     },
-                    programs
+                    programStageIds
                 );
             });
 
@@ -479,23 +478,24 @@ async function getRuleDataElements(
         .value();
 }
 
-async function getRulePrograms(
+async function getRuleProgramStageIds(
     compositionRoot: CompositionRoot,
     rule: SynchronizationRule,
     instance: Instance
 ): Promise<string[]> {
     const { objects } = await compositionRoot.metadata.list(
         {
             type: "programIndicators",
-            fields: { id: true, program: true },
+            fields: { id: true, program: { id: true, programStages: true } },
             filterRows: rule.metadataIds,
             paging: false,
         },
         instance
     );
 
-    return _(objects as Partial<ProgramIndicator>[])
-        .map(({ program }) => program?.id)
+    return _(objects as Partial<{ id: string; program: { id: string; programStages: Ref[] } }>[])
+        .map(({ program }) => program?.programStages.map(({ id }) => id))
+        .flatten()
         .compact()
         .uniq()
         .value();

tsconfig.json

@@ -1,4 +1,12 @@
 {
+    "ts-node": {
+        // these options are overrides used only by ts-node
+        // same as our --compilerOptions flag and our TS_NODE_COMPILER_OPTIONS environment variable
+        "compilerOptions": {
+            "module": "commonjs",
+            "typeRoots": ["./src/types", "./node_modules/@types"]
+        }
+    },
     "compilerOptions": {
         "target": "es2015",
         "allowJs": true,