Skip to content

Commit

Permalink
Merge pull request #275 from EyeSeeTea/development
Browse files Browse the repository at this point in the history 
Release 1.6.7
  • Loading branch information
@ifoche
ifoche authored Jul 3, 2024
2 parents 029e20d + b106c5c commit 40cb5d5
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 35 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "glass",
"description": "DHIS2 Glass App",
"version": "1.6.6",
"version": "1.6.7",
"license": "GPL-3.0",
"author": "EyeSeeTea team",
"homepage": ".",
Expand Down
34 changes: 29 additions & 5 deletions src/scripts/amr_agg_data_reset.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ function main() {
long: "period",
description: "The period to run amr-agg data reset for",
}),
batchId: option({
type: string,
long: "batchId",
description: "The batchId/dataset to run amr-agg data reset for",
}),
},
handler: async args => {
const api = getD2ApiFromArgs(args);
Expand All @@ -31,9 +36,22 @@ function main() {
if (!args.orgUnitId) throw new Error("OrgUnit is required");
const orgUnitId = args.orgUnitId;

//3. Set AMR-AGG dataset id.
//3. Get Batch Id to reset
if (!args.orgUnitId) throw new Error("OrgUnit is required");
const batchId = args.batchId;

//4. Set AMR-AGG dataset id.
const dataSetId = "CeQPmXgrhHF";

//5.Get all category combination values for given batchId
const batchCC = await api.models.categoryOptionCombos
.get({
fields: { id: true, name: true },
filter: { identifiable: { token: batchId } },
paging: false,
})
.getData();

//4. Get all data values for given country and period.
const dataSetValues = await api.dataValues
.getSet({
Expand All @@ -42,16 +60,22 @@ function main() {
period: [period],
})
.getData();
//4.b) Filter data values for given batchId
const filteredDataValues = dataSetValues.dataValues.filter(dv =>
batchCC.objects.map(coc => coc.id).includes(dv.attributeOptionCombo)
);

if (dataSetValues.dataValues.length === 0)
throw new Error(`No data values found for period ${period} and org unit ${orgUnitId}`);
if (filteredDataValues.length === 0)
throw new Error(
`No data values found for period ${period}, org unit ${orgUnitId} and batchId ${batchId}`
);

console.debug(
`${dataSetValues.dataValues.length} data values found for period ${period} and org unit ${orgUnitId}`
`${filteredDataValues.length} data values found for period ${period}, org unit ${orgUnitId} and batchId ${batchId}`
);

const updatedDataValues = {
dataValues: dataSetValues.dataValues.map(dataValue => {
dataValues: filteredDataValues.map(dataValue => {
return {
...dataValue,
value: "",
Expand Down
100 changes: 71 additions & 29 deletions src/scripts/amr_agg_data_validation.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { command, run } from "cmd-ts";
import { command, option, optional, run, string } from "cmd-ts";
import path from "path";
import _ from "lodash";
import { DataStoreClient } from "../data/data-store/DataStoreClient";
Expand All @@ -12,14 +12,19 @@ function main() {
description: "Show DHIS2 instance info",
args: {
...getApiUrlOptions(),
period: option({
type: optional(string),
long: "period",
description: "The period to run amr-agg data validation for",
}),
},
handler: async args => {
const api = getD2ApiFromArgs(args);
const instance = getInstance(args);
const dataStoreClient = new DataStoreClient(instance);

//1. Initialize all periods
const periods = ["2016", "2017", "2018", "2019", "2020", "2021", "2022", "2023"];
const periods = args.period ? [args.period] : ["2022", "2023"];

//2. Get all countries i.e org units of level 3.
const orgUnits = await api.models.organisationUnits
Expand All @@ -29,14 +34,17 @@ function main() {
paging: false,
})
.getData();
//Add Kosovo to the list of countries
//2.b) Add Kosovo to the list of countries
orgUnits.objects.push({
id: "I8AMbKhxlj9",
name: "Kosovo",
code: "601624",
});

//3. Get all data values for all countries and all periods
//3. Initialize all bacth Ids
const batchIds = ["DS1", "DS2", "DS3", "DS4", "DS5", "DS6"];

//4. Get all data values for all countries and all periods
const dataSetValues = await api.dataValues
.getSet({
dataSet: ["CeQPmXgrhHF"],
Expand All @@ -45,44 +53,78 @@ function main() {
})
.getData();

//5. Get all category option combos for containing all batchIds
const allBatchIdCC = await Promise.all(
batchIds.map(async batchId => {
const batchCC = await api.models.categoryOptionCombos
.get({
fields: { id: true, name: true },
filter: { identifiable: { token: batchId } },
paging: false,
})
.getData();

return batchCC.objects.map(coc => {
return {
batchId: batchId,
categoryComboId: coc.id,
};
});
})
);
const allBatchIdCategoryCombos = allBatchIdCC.flat();

//6. Group data values by orgUnit
const ouGroupedDataValues = _(dataSetValues.dataValues).groupBy("orgUnit");
const formattedResult = await Promise.all(
ouGroupedDataValues
.map(async (dataValues, orgUnitKey) => {
//7. Group data values by period
const periodGroupedDataValues = _(dataValues).groupBy("period");
const result = periodGroupedDataValues.map(async (dataValues, periodKey) => {
const result = periodGroupedDataValues.flatMap(async (dataValues, periodKey) => {
const country = orgUnits.objects.find(ou => ou.id === orgUnitKey)?.name;
//4. Get uploads for period and OU from datastore
const upload = await dataStoreClient
.getObjectsFilteredByProps<GlassUploads>(
DataStoreKeys.UPLOADS,
new Map<keyof GlassUploads, unknown>([
["module", "AVnpk4xiXGG"],
["orgUnit", orgUnitKey],
["period", periodKey],
])
)
.toPromise();

if (upload.length === 0)
console.debug(`ERROR: No upload found for period ${periodKey} and Country ${country}`);

return {
dataCount: dataValues.length,
period: periodKey,
country: country,
orgUnitId: orgUnitKey,
uploadStatuses: upload.map(u => u.status),
};

const dataValuesByBatch = batchIds.map(async batchId => {
//8. Get uploads for period, OU and batchId from datastore
const upload = await dataStoreClient
.getObjectsFilteredByProps<GlassUploads>(
DataStoreKeys.UPLOADS,
new Map<keyof GlassUploads, unknown>([
["module", "AVnpk4xiXGG"],
["orgUnit", orgUnitKey],
["period", periodKey],
["batchId", batchId],
])
)
.toPromise();

const currentBatchCC = allBatchIdCategoryCombos.filter(cc => cc.batchId === batchId);

//9. Filter data values by batch id
const dataValuesByBatchId = dataValues.filter(dv =>
currentBatchCC.map(cbcc => cbcc.categoryComboId).includes(dv.attributeOptionCombo)
);

return {
dataCount: dataValuesByBatchId.length,
period: periodKey,
country: country,
orgUnitId: orgUnitKey,
batchId: batchId,
uploadStatuses: upload.map(u => u.status),
};
});

return await Promise.all(dataValuesByBatch);
});
return await Promise.all(result.value());
})
.value()
);

const allAmrAggData = formattedResult.flat();
const allAmrAggData = formattedResult.flat().flat();

//5. Filter only corrupted data
//10. Filter only corrupted data
const corruptedAmrData = allAmrAggData.filter(
data =>
data.dataCount > 0 &&
Expand Down

0 comments on commit 40cb5d5

Please sign in to comment.