diff --git a/.travis.yml b/.travis.yml index a1e313ee1..9b7f5e961 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,39 @@ language: node_js node_js: - 12.13.0 +dist: bionic cache: directories: - "$HOME/.cache" before_install: - echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p + # Install python dependencies + - sudo apt-get update + - sudo apt-get install python3 python python3-setuptools docker.io docker-compose + # Install d2-docker + - git clone https://github.com/EyeSeeTea/d2-docker.git + - cd d2-docker/ + - sudo python3 setup.py install + - d2-docker --help + # Hack to not be prompted in the terminal + - sudo apt-get remove golang-docker-credential-helpers + # Start docker service + - sudo systemctl unmask docker.service + - sudo systemctl unmask docker.socket + - sudo systemctl start docker.service + # Login to docker + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + # Start docker service + - d2-docker start eyeseetea/dhis2-data:2.30-datasync-sender -d install: - yarn install --frozen-lockfile - yarn cy:verify - yarn build script: - - PORT=8081 REACT_APP_DHIS2_BASE_URL=http://dev2.eyeseetea.com:8083 REACT_APP_CYPRESS=true yarn start & + - PORT=8081 REACT_APP_DHIS2_BASE_URL=http://localhost:8080 REACT_APP_CYPRESS=true yarn start & + - yarn wait-on http-get://localhost:8080 - yarn wait-on http-get://localhost:8081 - - CYPRESS_EXTERNAL_API=http://dev2.eyeseetea.com:8083 CYPRESS_ROOT_URL=http://localhost:8081 yarn cy:e2e:run --key $CYPRESS_KEY + - CYPRESS_EXTERNAL_API=http://localhost:8080 CYPRESS_ROOT_URL=http://localhost:8081 yarn cy:e2e:run - kill $(jobs -p) || true addons: apt: diff --git a/i18n/en.pot b/i18n/en.pot index 85796cd12..0117803be 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,11 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2020-01-02T17:19:20.540Z\n" -"PO-Revision-Date: 2020-01-02T17:19:20.540Z\n" - -msgid "Search by name" -msgstr "" +"POT-Creation-Date: 2020-01-09T12:11:29.305Z\n" +"PO-Revision-Date: 2020-01-09T12:11:29.305Z\n" msgid "" msgstr "" @@ -38,6 +35,9 @@ msgstr "" msgid "Select" msgstr "" +msgid "Search by name" +msgstr "" + msgid "Name" msgstr "" @@ -311,6 +311,21 @@ msgstr "" msgid "Yes" msgstr "" +msgid "Excluded elements" +msgstr "" + +msgid "All events" +msgstr "" + +msgid "{{total}} selected events" +msgstr "" + +msgid "Category Option Combo" +msgstr "" + +msgid "All attribute category options" +msgstr "" + msgid "Start date" msgstr "" @@ -392,7 +407,7 @@ msgstr "" msgid "Network error {{error}}, check if server is up and CORS is enabled" msgstr "" -msgid "Metadata & Data Synchronization" +msgid "MetaData Synchronization" msgstr "" msgid "Metadata Synchronization History" @@ -535,40 +550,24 @@ msgstr "" msgid "Manual sync" msgstr "" -msgid "" -"Manually synchronise metadata like data elements, organisation units and " -"program indicators and groups and group sets." -msgstr "" - -msgid "Sync rules" -msgstr "" - -msgid "" -"Create, modify, delete, execute and schedule sync rules for metadata like " -"data elements, organisation units and program indicators and groups and " -"group sets." -msgstr "" - -msgid "History" -msgstr "" - -msgid "" -"View and analyse the status and results of the metadata manual syncs and " -"sync rules executions." -msgstr "" - msgid "" "Manually synchronise aggregated data by selecting the data sets, data " "elements or their groups and group sets together with the organisation " "unit, period and category options." msgstr "" +msgid "Sync rules" +msgstr "" + msgid "" "Create, modify, delete, execute and schedule sync rules for aggregated data " "by selecting the data sets, data elements or their groups and group sets " "together with the organisation unit, period and category options." msgstr "" +msgid "History" +msgstr "" + msgid "" "View and analyse the status and results of the aggregated data manual syncs " "and sync rules executions." @@ -590,6 +589,22 @@ msgid "" "rules executions." msgstr "" +msgid "" +"Manually synchronise metadata like data elements, organisation units and " +"program indicators and groups and group sets." +msgstr "" + +msgid "" +"Create, modify, delete, execute and schedule sync rules for metadata like " +"data elements, organisation units and program indicators and groups and " +"group sets." +msgstr "" + +msgid "" +"View and analyse the status and results of the metadata manual syncs and " +"sync rules executions." +msgstr "" + msgid "Deleted objects" msgstr "" diff --git a/icon.png b/icon.png index 8e07902c0..e586c198e 100644 Binary files a/icon.png and b/icon.png differ diff --git a/package.json b/package.json index 638bc7da1..0cc9cb776 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "metadata-synchronization", "description": "Advanced metadata & data synchronization utility", - "version": "0.6.0", + "version": "0.6.1", "license": "GPL-3.0", "author": "EyeSeeTea team", "homepage": ".", @@ -27,7 +27,7 @@ "d2": "^31.1.1", "d2-api": "^0.0.2-beta.9", "d2-manifest": "^1.0.0", - "d2-ui-components": "^1.0.1", + "d2-ui-components": "^1.0.2", "downshift": "^3.3.4", "enzyme": "^3.10.0", "enzyme-adapter-react-16": "^1.14.0", @@ -74,6 +74,7 @@ "@babel/core": "^7.7.5", "@babel/preset-typescript": "^7.7.4", "@types/cryptr": "^4.0.0", + "@types/file-saver": "^2.0.1", "@types/jest": "^24.0.23", "@types/lodash": "^4.14.149", "@types/node": "^12.12.14", @@ -101,7 +102,7 @@ "wait-on": "^3.3.0" }, "manifest.webapp": { - "name": "Metadata Synchronization", + "name": "MetaData Synchronization", "description": "Advanced metadata synchronization utility", "icons": { "48": "icon.png" diff --git a/public/index.html b/public/index.html index 938d64950..8d9ceb662 100644 --- a/public/index.html +++ b/public/index.html @@ -18,7 +18,7 @@ work correctly both with client-side routing and a non-root public URL. Learn how to configure a non-root public URL by running `npm run build`. --> - Metadata & Data Synchronization + MetaData Synchronization diff --git a/src/components/d2-objects-table/D2ObjectsTable.tsx b/src/components/d2-objects-table/D2ObjectsTable.tsx deleted file mode 100644 index 3d0c7102b..000000000 --- a/src/components/d2-objects-table/D2ObjectsTable.tsx +++ /dev/null @@ -1,126 +0,0 @@ -import i18n from "@dhis2/d2-i18n"; -import D2ApiModel from "d2-api/api/models"; -import { - ObjectsTable, - ObjectsTableProps, - ReferenceObject, - TableObject, - TableState, -} from "d2-ui-components"; -import _ from "lodash"; -import memoize from "nano-memoize"; -import React, { useEffect, useState } from "react"; -import { useD2ApiData } from "d2-api"; - -export interface D2ObjectsTableProps - extends Omit, "rows"> { - apiModel: InstanceType; - apiQuery?: Parameters["get"]>[0]; - transformObjects?(objects: T[]): T[]; -} - -const defaultState = { - sorting: { - field: "id" as const, - order: "asc" as const, - }, - pagination: { - page: 1, - pageSize: 25, - }, -}; - -const getAllIdentifiers = memoize( - async ( - _cacheKey: string, - search: string | undefined, - apiModel: InstanceType, - apiQuery: Parameters["get"]>[0] - ) => { - const { objects } = await apiModel - .get({ - ...apiQuery, - paging: false, - fields: { - id: true as true, - }, - filter: { - name: { ilike: search }, - ...apiQuery.filter, - }, - }) - .getData(); - return _.map(objects, "id"); - }, - { maxArgs: 2 } -); - -export function D2ObjectsTable( - props: D2ObjectsTableProps -) { - const { - apiModel, - apiQuery = { fields: {} }, - transformObjects = (objects: T[]) => objects, - initialState = defaultState, - onChange = _.noop, - ...rest - } = props; - const [ids, updateIds] = useState([]); - const [search, updateSearch] = useState(undefined); - const [sorting, updateSorting] = useState(initialState.sorting || defaultState.sorting); - const [pagination, updatePagination] = useState( - initialState.pagination || defaultState.pagination - ); - - const { loading, data, error, refetch } = useD2ApiData(); - - useEffect(() => { - getAllIdentifiers(apiModel.modelName, search, apiModel, apiQuery).then(updateIds); - }, [apiModel, apiQuery, search]); - - useEffect( - () => - refetch( - apiModel.get({ - order: `${sorting.field}:i${sorting.order}`, - page: pagination.page, - pageSize: pagination.pageSize, - ...apiQuery, - filter: { - name: { ilike: search }, - ...apiQuery.filter, - }, - }) - ), - [apiModel, apiQuery, refetch, sorting, pagination, search] - ); - - if (error) return

{"Error: " + JSON.stringify(error)}

; - - // @ts-ignore @tokland How do we handle non-paginated inference here? - const { objects, pager } = data || { objects: [], pager: undefined }; - - const handleTableChange = (tableState: TableState) => { - const { sorting, pagination } = tableState; - updateSorting(sorting); - updatePagination(pagination); - onChange(tableState); - }; - - return ( - - rows={transformObjects(objects)} - onChangeSearch={updateSearch} - initialState={initialState} - searchBoxLabel={i18n.t("Search by name")} - pagination={pager} - onChange={handleTableChange} - ids={ids} - loading={loading} - {...rest} - /> - ); -} - -export default D2ObjectsTable; diff --git a/src/components/metadata-table/MetadataTable.tsx b/src/components/metadata-table/MetadataTable.tsx index c70951c41..678f0250c 100644 --- a/src/components/metadata-table/MetadataTable.tsx +++ b/src/components/metadata-table/MetadataTable.tsx @@ -1,27 +1,34 @@ import { Checkbox, FormControlLabel, makeStyles } from "@material-ui/core"; -import { useD2, useD2Api, D2Api } from "d2-api"; +import DoneAllIcon from "@material-ui/icons/DoneAll"; +import { useD2, useD2Api, useD2ApiData } from "d2-api"; import D2ApiModel from "d2-api/api/models"; -import { DatePicker, ReferenceObject, TableState } from "d2-ui-components"; +import { + DatePicker, + ObjectsTable, + ObjectsTableProps, + ReferenceObject, + TableSelection, + TableSorting, + TableState, +} from "d2-ui-components"; import _ from "lodash"; import moment from "moment"; -import memoize from "nano-memoize"; import React, { ChangeEvent, useEffect, useMemo, useState } from "react"; import i18n from "../../locales"; import { getOrgUnitSubtree } from "../../logic/utils"; import { D2Model, DataElementModel } from "../../models/d2Model"; -import { d2ModelFactory } from "../../models/d2ModelFactory"; import { D2 } from "../../types/d2"; import { NamedRef } from "../../types/synchronization"; import { d2BaseModelFields, MetadataType } from "../../utils/d2"; -import D2ObjectsTable, { D2ObjectsTableProps } from "../d2-objects-table/D2ObjectsTable"; import Dropdown from "../dropdown/Dropdown"; -import DoneAllIcon from "@material-ui/icons/DoneAll"; +import { getAllIdentifiers, getFilterData } from "./utils"; -interface MetadataTableProps - extends Omit, "columns" | "apiModel"> { +interface MetadataTableProps extends Omit, "rows" | "columns"> { models: typeof D2Model[]; - selection?: string[]; - notifyNewSelection?(selection: string[]): void; + selectedIds?: string[]; + excludedIds?: string[]; + notifyNewSelection?(selectedIds: string[], excludedIds: string[]): void; + childrenKeys?: string[]; } const useStyles = makeStyles({ @@ -31,28 +38,6 @@ const useStyles = makeStyles({ }, }); -const getData = memoize( - (modelName: string, type: "group" | "level", d2: D2, api: D2Api) => - d2ModelFactory(d2, modelName) - .getApiModel(api) - .get({ - paging: false, - fields: - type === "group" - ? { - id: true as true, - name: true as true, - } - : { - name: true as true, - level: true as true, - }, - order: type === "group" ? undefined : `level:iasc`, - }) - .getData(), - { maxArgs: 2 } -); - interface FiltersState { lastUpdated: Date | null; group: string; @@ -68,10 +53,23 @@ interface FiltersState { }[]; } +const initialState = { + sorting: { + field: "displayName" as const, + order: "asc" as const, + }, + pagination: { + page: 1, + pageSize: 25, + }, +}; + const MetadataTable: React.FC = ({ models, - selection = [], + selectedIds = [], + excludedIds = [], notifyNewSelection = _.noop, + childrenKeys = [], ...rest }) => { const d2 = useD2() as D2; @@ -79,6 +77,10 @@ const MetadataTable: React.FC = ({ const classes = useStyles({}); const [model, updateModel] = useState(() => models[0] || DataElementModel); + const [ids, updateIds] = useState([]); + const [search, updateSearch] = useState(undefined); + const [sorting, updateSorting] = useState>(initialState.sorting); + const [pagination, updatePagination] = useState(initialState.pagination); const [filters, updateFilters] = useState({ lastUpdated: null, group: "", @@ -88,28 +90,6 @@ const MetadataTable: React.FC = ({ showOnlySelected: false, }); - useEffect(() => { - if (model && model.getGroupFilterName()) { - getData(model.getGroupFilterName(), "group", d2, api).then(({ objects }) => - updateFilters(state => ({ ...state, groupData: objects })) - ); - } - - if (model && model.getLevelFilterName()) { - getData(model.getLevelFilterName(), "level", d2, api).then(({ objects }) => { - updateFilters(state => ({ - ...state, - levelData: objects.map(e => ({ - //@ts-ignore Bug in inference (level not detected) - id: e.level, - //@ts-ignore Bug in inference (level not detected) - name: `${e.level}. ${e.name}`, - })), - })); - }); - } - }, [d2, api, model]); - const changeDropdownFilter = (event: ChangeEvent) => { if (models.length === 0) throw new Error("You need to provide at least one model"); const model = @@ -133,69 +113,31 @@ const MetadataTable: React.FC = ({ updateFilters(state => ({ ...state, showOnlySelected: event.target.checked })); }; - const handleTableChange = (tableState: TableState) => { - const { selection } = tableState; - notifyNewSelection(selection); - }; - - const selectChildren = async (selectedOUs: NamedRef[]) => { + const selectOrgUnitChildren = async (selectedOUs: NamedRef[]) => { const ids = new Set(); for (const selectedOU of selectedOUs) { const subtree = await getOrgUnitSubtree(d2 as D2, selectedOU.id); subtree.forEach(id => ids.add(id)); } - notifyNewSelection([...selection, ...Array.from(ids)]); + notifyNewSelection([...selectedIds, ...Array.from(ids)], excludedIds); }; const addToSelection = (items: NamedRef[]) => { const ids = items.map(({ id }) => id); - const oldSelection = _.difference(selection, ids); - const newSelection = _.difference(ids, selection); + const oldSelection = _.difference(selectedIds, ids); + const newSelection = _.difference(ids, selectedIds); - notifyNewSelection([...oldSelection, ...newSelection]); + notifyNewSelection([...oldSelection, ...newSelection], excludedIds); }; - const groupTypes = models.map(model => ({ - id: model.getMetadataType(), - name: model.getD2Model(d2).displayName, - })); - - const apiQuery: Parameters["get"]>[0] = useMemo(() => { - // TODO: Update in d2-api type definition with field accessor - const query: Parameters["get"]>[0] = { - fields: model ? model.getFields() : d2BaseModelFields, - filter: { - lastUpdated: filters.lastUpdated - ? { ge: moment(filters.lastUpdated).format("YYYY-MM-DD") } - : undefined, - id: filters.showOnlySelected ? { in: selection } : undefined, - ...model.getApiModelFilters(), - }, - }; - - if (query.filter && model.getGroupFilterName()) { - query.filter[`${model.getGroupFilterName()}.id`] = { eq: filters.group }; - } - - if (query.filter && model.getLevelFilterName()) { - query.filter["level"] = { eq: filters.level }; - } - - return query; - }, [ - model, - selection, - filters.lastUpdated, - filters.showOnlySelected, - filters.group, - filters.level, - ]); - const filterComponents = _.compact([ models.length > 1 && ( ({ + id: model.getMetadataType(), + name: model.getD2Model(d2).displayName, + }))} onChange={changeDropdownFilter} value={model.getMetadataType()} label={i18n.t("Metadata type")} @@ -256,7 +198,7 @@ const MetadataTable: React.FC = ({ name: "select-children", text: i18n.t("Select with children subtree"), multiple: true, - onClick: selectChildren, + onClick: selectOrgUnitChildren, icon: , isActive: () => { return model.getMetadataType() === "organisationUnit"; @@ -272,26 +214,151 @@ const MetadataTable: React.FC = ({ }, ]; - const initialState = { - sorting: { - field: "displayName" as const, - order: "asc" as const, - }, + const apiModel = model.getApiModel(api); + const apiQuery = useMemo(() => { + const query: Parameters["get"]>[0] = { + fields: model ? model.getFields() : d2BaseModelFields, + filter: { + lastUpdated: filters.lastUpdated + ? { ge: moment(filters.lastUpdated).format("YYYY-MM-DD") } + : undefined, + id: filters.showOnlySelected ? { in: selectedIds } : undefined, + ...model.getApiModelFilters(), + }, + }; + + if (query.filter && model.getGroupFilterName()) { + query.filter[`${model.getGroupFilterName()}.id`] = { eq: filters.group }; + } + + if (query.filter && model.getLevelFilterName()) { + query.filter["level"] = { eq: filters.level }; + } + + return query; + }, [ + model, + selectedIds, + filters.lastUpdated, + filters.showOnlySelected, + filters.group, + filters.level, + ]); + + const { loading, data, error, refetch } = useD2ApiData(); + + useEffect(() => { + getAllIdentifiers(apiModel.modelName, search, apiModel, apiQuery).then(updateIds); + }, [apiModel, apiQuery, search]); + + useEffect( + () => + refetch( + apiModel.get({ + order: `${sorting.field}:i${sorting.order}`, + page: pagination.page, + pageSize: pagination.pageSize, + ...apiQuery, + filter: { + name: { ilike: search }, + ...apiQuery.filter, + }, + }) + ), + [apiModel, apiQuery, refetch, sorting, pagination, search] + ); + + useEffect(() => { + if (model && model.getGroupFilterName()) { + getFilterData(model.getGroupFilterName(), "group", d2, api).then(({ objects }) => + updateFilters(state => ({ ...state, groupData: objects })) + ); + } + + if (model && model.getLevelFilterName()) { + getFilterData(model.getLevelFilterName(), "level", d2, api).then(({ objects }) => { + // Inference does not work for orgUnits here + const levels = (objects as unknown) as { name: string; level: number }[]; + updateFilters(state => ({ + ...state, + levelData: levels.map(({ name, level }) => ({ + id: String(level), + name: `${level}. ${name}`, + })), + })); + }); + } + }, [d2, api, model]); + + if (error) return

{"Error: " + JSON.stringify(error)}

; + + const { objects, pager } = data || { objects: [], pager: undefined }; + const rows = model.getApiModelTransform()(objects); + + const handleTableChange = (tableState: TableState) => { + const { sorting, pagination, selection } = tableState; + + const included = _.reject(selection, { indeterminate: true }).map(({ id }) => id); + const newlySelectedIds = _.difference(included, selectedIds); + const newlyUnselectedIds = _.difference(selectedIds, included); + + const childrenOfNewlySelected = _(rows) + .filter(({ id }) => !!newlySelectedIds.includes(id)) + .map(row => (_.values(_.pick(row, childrenKeys)) as unknown) as MetadataType) + .flattenDeep() + .map(({ id }) => id) + .value(); + + const excluded = _(excludedIds) + .union(newlyUnselectedIds) + .difference(childrenOfNewlySelected) + .filter(id => !_.find(rows, { id })) + .value(); + + updateSorting(sorting); + updatePagination(pagination); + notifyNewSelection(included, excluded); }; + const exclusion = excludedIds.map(id => ({ id })); + const selection = selectedIds.map(id => ({ + id, + checked: true, + indeterminate: false, + })); + + const childrenSelection: TableSelection[] = _(rows) + .intersectionBy(selection, "id") + .map(row => (_.values(_.pick(row, childrenKeys)) as unknown) as MetadataType) + .flattenDeep() + .differenceBy(selection, "id") + .differenceBy(exclusion, "id") + .map(({ id }) => { + return { + id, + checked: true, + indeterminate: !_.find(selection, { id }), + } as TableSelection; + }) + .value(); + return ( - - apiModel={model.getApiModel(api)} - apiQuery={apiQuery} - transformObjects={model.getApiModelTransform()} + + rows={rows} columns={model.getColumns()} details={model.getDetails()} + onChangeSearch={updateSearch} + initialState={initialState} + searchBoxLabel={i18n.t("Search by name")} + pagination={pager} + onChange={handleTableChange} + ids={ids} + loading={loading} + selection={[...selection, ...childrenSelection]} + childrenKeys={childrenKeys} filterComponents={filterComponents} forceSelectionColumn={true} actions={actions} - selection={selection} - onChange={handleTableChange} - initialState={initialState} {...rest} /> ); diff --git a/src/components/metadata-table/utils.tsx b/src/components/metadata-table/utils.tsx new file mode 100644 index 000000000..fc7ac6282 --- /dev/null +++ b/src/components/metadata-table/utils.tsx @@ -0,0 +1,53 @@ +import { D2Api } from "d2-api"; +import D2ApiModel from "d2-api/api/models"; +import memoize from "nano-memoize"; +import { d2ModelFactory } from "../../models/d2ModelFactory"; +import { D2 } from "../../types/d2"; +import _ from "lodash"; + +export const getFilterData = memoize( + (modelName: string, type: "group" | "level", d2: D2, api: D2Api) => + d2ModelFactory(d2, modelName) + .getApiModel(api) + .get({ + paging: false, + fields: + type === "group" + ? { + id: true as true, + name: true as true, + } + : { + name: true as true, + level: true as true, + }, + order: type === "group" ? undefined : `level:iasc`, + }) + .getData(), + { maxArgs: 2 } +); + +export const getAllIdentifiers = memoize( + async ( + _cacheKey: string, + search: string | undefined, + apiModel: InstanceType, + apiQuery: Parameters["get"]>[0] + ) => { + const { objects } = await apiModel + .get({ + ...apiQuery, + paging: false, + fields: { + id: true as true, + }, + filter: { + name: { ilike: search }, + ...apiQuery.filter, + }, + }) + .getData(); + return _.map(objects, "id"); + }, + { maxArgs: 2 } +); diff --git a/src/components/sync-wizard/common/MetadataSelectionStep.tsx b/src/components/sync-wizard/common/MetadataSelectionStep.tsx index 02eb02cfe..3e5ee07b5 100644 --- a/src/components/sync-wizard/common/MetadataSelectionStep.tsx +++ b/src/components/sync-wizard/common/MetadataSelectionStep.tsx @@ -40,7 +40,7 @@ export default function MetadataSelectionStep(props: SyncWizardStepProps) { const [metadataIds, updateMetadataIds] = useState([]); const snackbar = useSnackbar(); - const changeSelection = (newMetadataIds: string[]) => { + const changeSelection = (newMetadataIds: string[], newExclusionIds: string[]) => { const additions = _.difference(newMetadataIds, metadataIds); if (additions.length > 0) { snackbar.info( @@ -61,14 +61,15 @@ export default function MetadataSelectionStep(props: SyncWizardStepProps) { ); } - onChange(syncRule.updateMetadataIds(newMetadataIds)); + onChange(syncRule.updateMetadataIds(newMetadataIds).updateExcludedIds(newExclusionIds)); updateMetadataIds(newMetadataIds); }; return ( diff --git a/src/components/sync-wizard/common/SchedulerStep.jsx b/src/components/sync-wizard/common/SchedulerStep.jsx index 4662348e8..096da55ed 100644 --- a/src/components/sync-wizard/common/SchedulerStep.jsx +++ b/src/components/sync-wizard/common/SchedulerStep.jsx @@ -8,11 +8,11 @@ import { Toggle } from "../../toggle/Toggle"; import isValidCronExpression from "../../../utils/validCronExpression"; const cronExpressions = [ - { displayName: i18n.t("Every day"), id: "0 0 12 ? * *" }, - { displayName: i18n.t("Every month"), id: "0 0 12 1 1/1 ?" }, - { displayName: i18n.t("Every three months"), id: "0 0 12 1 1/3 ?" }, - { displayName: i18n.t("Every six months"), id: "0 0 12 1 1/6 ?" }, - { displayName: i18n.t("Every year"), id: "0 0 12 1 1 ?" }, + { displayName: i18n.t("Every day"), id: "0 0 0 ? * *" }, + { displayName: i18n.t("Every month"), id: "0 0 0 1 1/1 ?" }, + { displayName: i18n.t("Every three months"), id: "0 0 0 1 1/3 ?" }, + { displayName: i18n.t("Every six months"), id: "0 0 0 1 1/6 ?" }, + { displayName: i18n.t("Every year"), id: "0 0 0 1 1 ?" }, ]; const SchedulerStep = ({ syncRule, onChange }) => { diff --git a/src/components/sync-wizard/common/SummaryStep.jsx b/src/components/sync-wizard/common/SummaryStep.jsx index d61dc84b7..95f61a5fc 100644 --- a/src/components/sync-wizard/common/SummaryStep.jsx +++ b/src/components/sync-wizard/common/SummaryStep.jsx @@ -2,7 +2,6 @@ import i18n from "@dhis2/d2-i18n"; import { Button, LinearProgress, withStyles } from "@material-ui/core"; import { useD2, useD2Api } from "d2-api"; import { ConfirmationDialog, useSnackbar, withLoading } from "d2-ui-components"; -import FileSaver from "file-saver"; import _ from "lodash"; import moment from "moment"; import PropTypes from "prop-types"; @@ -11,7 +10,12 @@ import { AggregatedSync } from "../../../logic/sync/aggregated"; import { EventsSync } from "../../../logic/sync/events"; import { MetadataSync } from "../../../logic/sync/metadata"; import { getBaseUrl } from "../../../utils/d2"; -import { availablePeriods, getMetadata } from "../../../utils/synchronization"; +import { + availablePeriods, + cleanOrgUnitPaths, + getMetadata, + requestJSONDownload, +} from "../../../utils/synchronization"; import { getValidationMessages } from "../../../utils/validations"; import { getInstances } from "./InstanceSelectionStep"; @@ -86,18 +90,18 @@ const SaveStep = ({ syncRule, classes, onCancel, loading }) => { const { SyncClass } = config[syncRule.type]; loading.show(true, "Generating JSON file"); - - const sync = new SyncClass(d2, api, syncRule.toBuilder()); - const payload = await sync.buildPayload(); - - const json = JSON.stringify(payload, null, 4); - const blob = new Blob([json], { type: "application/json" }); - FileSaver.saveAs(blob, `${syncRule.type}-sync-${moment().format("YYYYMMDDHHmm")}.json`); + requestJSONDownload(SyncClass, syncRule, d2, api); loading.reset(); }; useEffect(() => { - getMetadata(getBaseUrl(d2), syncRule.metadataIds, "id,name").then(updateMetadata); + const ids = [ + ...syncRule.metadataIds, + ...syncRule.excludedIds, + ...syncRule.dataSyncAttributeCategoryOptions, + ...cleanOrgUnitPaths(syncRule.dataSyncOrgUnitPaths), + ]; + getMetadata(getBaseUrl(d2), ids, "id,name").then(updateMetadata); getInstances(d2).then(setInstanceOptions); }, [d2, syncRule]); @@ -121,18 +125,67 @@ const SaveStep = ({ syncRule, classes, onCancel, loading }) => { - {_.keys(metadata).map(metadataType => ( + {_.keys(metadata).map(metadataType => { + const items = metadata[metadataType].filter( + ({ id }) => !syncRule.excludedIds.includes(id) + ); + return ( + items.length > 0 && ( + +
    + {items.map(({ id, name }) => ( + + ))} +
+ + ) + ); + })} + + {syncRule.excludedIds.length > 0 && (
    - {metadata[metadataType].map(({ id, name }) => ( - - ))} + {syncRule.excludedIds.map(id => { + const element = _(metadata) + .values() + .flatten() + .find({ id }); + + return ( + + ); + })}
- ))} + )} + + {syncRule.type === "events" && ( + + )} + + {syncRule.dataSyncAllAttributeCategoryOptions && ( + + )} {syncRule.type !== "metadata" && ( {
)} @@ -151,7 +204,7 @@ const SaveStep = ({ syncRule, classes, onCancel, loading }) => {
)} diff --git a/src/components/sync-wizard/data/EventsSelectionStep.tsx b/src/components/sync-wizard/data/EventsSelectionStep.tsx index d39fac095..4d3a4b3e4 100644 --- a/src/components/sync-wizard/data/EventsSelectionStep.tsx +++ b/src/components/sync-wizard/data/EventsSelectionStep.tsx @@ -11,6 +11,10 @@ import Dropdown from "../../dropdown/Dropdown"; import { Toggle } from "../../toggle/Toggle"; import { SyncWizardStepProps } from "../Steps"; +interface ProgramEventObject extends ProgramEvent { + [key: string]: any; +} + export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardStepProps) { const d2 = useD2(); const api = useD2Api(); @@ -56,7 +60,7 @@ export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardSt const handleTableChange = (tableState: TableState) => { const { selection } = tableState; - onChange(syncRule.updateDataSyncEvents(selection)); + onChange(syncRule.updateDataSyncEvents(selection.map(({ id }) => id))); }; const updateSyncAll = (value: boolean) => { @@ -135,7 +139,7 @@ export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardSt onValueChange={updateSyncAll} /> {!syncRule.dataSyncAllEvents && ( - + rows={filteredObjects} loading={objects === undefined} columns={[...columns, ...additionalColumns]} @@ -143,7 +147,7 @@ export default function EventsSelectionStep({ syncRule, onChange }: SyncWizardSt actions={actions} forceSelectionColumn={true} onChange={handleTableChange} - selection={syncRule.dataSyncEvents ?? []} + selection={syncRule.dataSyncEvents?.map(id => ({ id })) ?? []} filterComponents={filterComponents} /> )} diff --git a/src/logic/sync/aggregated.ts b/src/logic/sync/aggregated.ts index 97d9a204b..26f05cb1b 100644 --- a/src/logic/sync/aggregated.ts +++ b/src/logic/sync/aggregated.ts @@ -2,7 +2,6 @@ import _ from "lodash"; import memoize from "nano-memoize"; import Instance from "../../models/instance"; import { DataImportResponse } from "../../types/d2"; -import { DataValue } from "../../types/synchronization"; import { buildMetadataDictionary, cleanDataImportResponse, @@ -17,7 +16,7 @@ export class AggregatedSync extends GenericSync { "id,dataElements[id,name]dataSetElements[:all,dataElement[id,name]],dataElementGroups[id,dataElements[id,name]],name"; public buildPayload = memoize(async () => { - const { dataParams = {} } = this.builder; + const { dataParams = {}, excludedIds = [] } = this.builder; const { dataSets = [], dataElementGroups = [], @@ -32,7 +31,6 @@ export class AggregatedSync extends GenericSync { ); // Retrieve direct data values from dataSets and dataElementGroups - //@ts-ignore const { dataValues: directDataValues = [] } = await getAggregatedData( this.api, dataParams, @@ -44,7 +42,6 @@ export class AggregatedSync extends GenericSync { ); // Retrieve candidate data values from dataElements - //@ts-ignore const { dataValues: candidateDataValues = [] } = await getAggregatedData( this.api, dataParams, @@ -53,14 +50,15 @@ export class AggregatedSync extends GenericSync { ); // Retrieve indirect data values from dataElements - const indirectDataValues = _.filter(candidateDataValues, ({ dataElement }) => - _.find(dataElements, { id: dataElement }) + const indirectDataValues = _.filter( + candidateDataValues, + ({ dataElement }) => !!_.find(dataElements, { id: dataElement }) ); - const dataValues = _.uniqWith( - [...directDataValues, ...indirectDataValues], - _.isEqual - ) as DataValue[]; + const dataValues = _([...directDataValues, ...indirectDataValues]) + .uniqWith(_.isEqual) + .reject(({ dataElement }) => excludedIds.includes(dataElement)) + .value(); return { dataValues }; }); diff --git a/src/logic/sync/generic.ts b/src/logic/sync/generic.ts index c8fe38169..f46c5b4c4 100644 --- a/src/logic/sync/generic.ts +++ b/src/logic/sync/generic.ts @@ -18,6 +18,11 @@ import { SyncRuleType, } from "../../types/synchronization"; import { getMetadata } from "../../utils/synchronization"; +import { AggregatedSync } from "./aggregated"; +import { EventsSync } from "./events"; +import { MetadataSync } from "./metadata"; + +export type SyncronizationClass = typeof MetadataSync | typeof AggregatedSync | typeof EventsSync; export abstract class GenericSync { protected readonly d2: D2; diff --git a/src/models/d2Model.ts b/src/models/d2Model.ts index 553d0fa5a..f58581fc1 100644 --- a/src/models/d2Model.ts +++ b/src/models/d2Model.ts @@ -1,4 +1,4 @@ -import { D2Api, D2DataSetSchema, D2ProgramSchema, SelectedPick } from "d2-api"; +import { D2Api, D2DataSetSchema, D2ModelSchemas, D2ProgramSchema, SelectedPick } from "d2-api"; import D2ApiModel from "d2-api/api/models"; import { ObjectsTableDetailField, TableColumn } from "d2-ui-components"; import { isValidUid } from "d2/uid"; @@ -29,7 +29,7 @@ import { export abstract class D2Model { // Metadata Type should be defined on subclasses protected static metadataType: string; - protected static collectionName: string; + protected static collectionName: keyof D2ModelSchemas; protected static groupFilterName: string; protected static levelFilterName: string; @@ -41,7 +41,7 @@ export abstract class D2Model { protected static details = d2BaseModelDetails; protected static fields = d2BaseModelFields; protected static initialSorting = ["name", "asc"]; - protected static modelTransform: Function | undefined = undefined; + protected static modelTransform: Function = (objects: any[]) => objects; protected static modelFilters: any = {}; // List method should be executed by a wrapper to preserve static context binding @@ -87,7 +87,9 @@ export abstract class D2Model { } public static getApiModel(api: D2Api): InstanceType { - const modelCollection = api.models as { [key: string]: InstanceType }; + const modelCollection = api.models as { + [ModelName in keyof D2ModelSchemas]: D2ApiModel; + }; return modelCollection[this.collectionName]; } @@ -144,7 +146,7 @@ export abstract class D2Model { export class OrganisationUnitModel extends D2Model { protected static metadataType = "organisationUnit"; - protected static collectionName = "organisationUnits"; + protected static collectionName = "organisationUnits" as const; protected static groupFilterName = "organisationUnitGroups"; protected static levelFilterName = "organisationUnitLevels"; @@ -177,7 +179,7 @@ export class OrganisationUnitModel extends D2Model { export class OrganisationUnitGroupModel extends D2Model { protected static metadataType = "organisationUnitGroup"; - protected static collectionName = "organisationUnitGroups"; + protected static collectionName = "organisationUnitGroups" as const; protected static excludeRules = ["legendSets", "organisationUnits.organisationUnitGroups"]; protected static includeRules = [ @@ -191,7 +193,7 @@ export class OrganisationUnitGroupModel extends D2Model { export class OrganisationUnitGroupSetModel extends D2Model { protected static metadataType = "organisationUnitGroupSet"; - protected static collectionName = "organisationUnitGroupSets"; + protected static collectionName = "organisationUnitGroupSets" as const; protected static excludeRules = [ "organisationUnitGroups.organisationUnitGroupSets", @@ -207,12 +209,12 @@ export class OrganisationUnitGroupSetModel extends D2Model { export class OrganisationUnitLevelModel extends D2Model { protected static metadataType = "organisationUnitLevel"; - protected static collectionName = "organisationUnitLevels"; + protected static collectionName = "organisationUnitLevels" as const; } export class DataElementModel extends D2Model { protected static metadataType = "dataElement"; - protected static collectionName = "dataElements"; + protected static collectionName = "dataElements" as const; protected static groupFilterName = "dataElementGroups"; protected static includeRules = [ @@ -249,7 +251,7 @@ export class ProgramDataElementModel extends DataElementModel { export class DataElementGroupModel extends D2Model { protected static metadataType = "dataElementGroup"; - protected static collectionName = "dataElementGroups"; + protected static collectionName = "dataElementGroups" as const; protected static fields = dataElementGroupFields; protected static excludeRules = ["legendSets", "dataElements.dataElementGroups"]; @@ -264,7 +266,7 @@ export class DataElementGroupModel extends D2Model { export class DataElementGroupSetModel extends D2Model { protected static metadataType = "dataElementGroupSet"; - protected static collectionName = "dataElementGroupSets"; + protected static collectionName = "dataElementGroupSets" as const; protected static fields = dataElementGroupSetFields; protected static excludeRules = [ @@ -281,7 +283,7 @@ export class DataElementGroupSetModel extends D2Model { export class DataSetModel extends D2Model { protected static metadataType = "dataSet"; - protected static collectionName = "dataSets"; + protected static collectionName = "dataSets" as const; protected static fields = dataSetFields; protected static modelTransform = ( @@ -296,7 +298,7 @@ export class DataSetModel extends D2Model { export class ProgramModel extends D2Model { protected static metadataType = "program"; - protected static collectionName = "programs"; + protected static collectionName = "programs" as const; protected static fields = programFields; protected static modelTransform = ( @@ -321,7 +323,7 @@ export class ProgramModel extends D2Model { export class IndicatorModel extends D2Model { protected static metadataType = "indicator"; - protected static collectionName = "indicators"; + protected static collectionName = "indicators" as const; protected static groupFilterName = "indicatorGroups"; protected static excludeRules = ["dataSets", "programs"]; @@ -337,7 +339,7 @@ export class IndicatorModel extends D2Model { export class IndicatorGroupModel extends D2Model { protected static metadataType = "indicatorGroup"; - protected static collectionName = "indicatorGroups"; + protected static collectionName = "indicatorGroups" as const; protected static excludeRules = ["legendSets", "indicators.indicatorGroups"]; protected static includeRules = [ @@ -351,7 +353,7 @@ export class IndicatorGroupModel extends D2Model { export class IndicatorGroupSetModel extends D2Model { protected static metadataType = "indicatorGroupSet"; - protected static collectionName = "indicatorGroupSets"; + protected static collectionName = "indicatorGroupSets" as const; protected static excludeRules = [ "indicatorGroups.indicatorGroupSets", @@ -367,7 +369,7 @@ export class IndicatorGroupSetModel extends D2Model { export class ProgramIndicatorModel extends D2Model { protected static metadataType = "programIndicator"; - protected static collectionName = "programIndicators"; + protected static collectionName = "programIndicators" as const; protected static groupFilterName = "programIndicatorGroups"; protected static excludeRules = ["programs"]; @@ -381,7 +383,7 @@ export class ProgramIndicatorModel extends D2Model { export class ProgramIndicatorGroupModel extends D2Model { protected static metadataType = "programIndicatorGroup"; - protected static collectionName = "programIndicatorGroups"; + protected static collectionName = "programIndicatorGroups" as const; protected static excludeRules = ["legendSets", "programIndicators.programIndicatorGroups"]; protected static includeRules = [ @@ -393,7 +395,7 @@ export class ProgramIndicatorGroupModel extends D2Model { export class ProgramRuleModel extends D2Model { protected static metadataType = "programRule"; - protected static collectionName = "programRules"; + protected static collectionName = "programRules" as const; protected static excludeRules = []; protected static includeRules = ["attributes", "programRuleActions"]; @@ -401,7 +403,7 @@ export class ProgramRuleModel extends D2Model { export class ProgramRuleVariableModel extends D2Model { protected static metadataType = "programRuleVariable"; - protected static collectionName = "programRuleVariables"; + protected static collectionName = "programRuleVariables" as const; protected static excludeRules = []; protected static includeRules = ["attributes"]; @@ -409,7 +411,7 @@ export class ProgramRuleVariableModel extends D2Model { export class ValidationRuleModel extends D2Model { protected static metadataType = "validationRule"; - protected static collectionName = "validationRules"; + protected static collectionName = "validationRules" as const; protected static groupFilterName = "validationRuleGroups"; protected static excludeRules = ["legendSets"]; @@ -422,7 +424,7 @@ export class ValidationRuleModel extends D2Model { export class ValidationRuleGroupModel extends D2Model { protected static metadataType = "validationRuleGroup"; - protected static collectionName = "validationRuleGroups"; + protected static collectionName = "validationRuleGroups" as const; protected static excludeRules = ["legendSets", "validationRules.validationRuleGroups"]; protected static includeRules = ["attributes", "validationRules", "validationRules.attributes"]; @@ -431,6 +433,6 @@ export class ValidationRuleGroupModel extends D2Model { export function defaultModel(pascalCaseModelName: string): any { return class DefaultModel extends D2Model { protected static metadataType = pascalCaseModelName; - protected static collectionName = pascalCaseModelName; + protected static collectionName = pascalCaseModelName as keyof D2ModelSchemas; }; } diff --git a/src/models/syncRule.ts b/src/models/syncRule.ts index 16f709b84..c83bcf56e 100644 --- a/src/models/syncRule.ts +++ b/src/models/syncRule.ts @@ -74,6 +74,10 @@ export default class SyncRule { return this.syncRule.builder.metadataIds; } + public get excludedIds(): string[] { + return this.syncRule.builder.excludedIds; + } + public get dataSyncAttributeCategoryOptions(): string[] { return this.syncRule.builder.dataParams?.attributeCategoryOptions ?? []; } @@ -167,6 +171,7 @@ export default class SyncRule { builder: { targetInstances: [], metadataIds: [], + excludedIds: [], dataParams: { strategy: "NEW_AND_UPDATES", allAttributeCategoryOptions: true, @@ -195,7 +200,7 @@ export default class SyncRule { } public static createOnDemand(type: SyncRuleType = "metadata"): SyncRule { - return SyncRule.create(type).updateName("On-demand"); + return SyncRule.create(type).updateName("__MANUAL__"); } public static build(syncRule: SynchronizationRule | undefined): SyncRule { @@ -256,7 +261,13 @@ export default class SyncRule { } public toBuilder() { - return _.pick(this, ["metadataIds", "targetInstances", "syncParams", "dataParams"]); + return _.pick(this, [ + "metadataIds", + "excludedIds", + "targetInstances", + "syncParams", + "dataParams", + ]); } public updateId(id: string): SyncRule { @@ -297,6 +308,16 @@ export default class SyncRule { }); } + public updateExcludedIds(excludedIds: string[]): SyncRule { + return SyncRule.build({ + ...this.syncRule, + builder: { + ...this.syncRule.builder, + excludedIds, + }, + }); + } + public updateDataSyncAttributeCategoryOptions(attributeCategoryOptions?: string[]): SyncRule { return SyncRule.build({ ...this.syncRule, @@ -455,7 +476,7 @@ export default class SyncRule { } public isOnDemand() { - return this.name === "On-demand"; + return this.name === "__MANUAL__"; } public isVisibleToUser(userInfo: UserInfo, permission: "READ" | "WRITE" = "READ") { diff --git a/src/pages/app/App.jsx b/src/pages/app/App.jsx index 821e3651f..e89edc0ea 100644 --- a/src/pages/app/App.jsx +++ b/src/pages/app/App.jsx @@ -102,9 +102,7 @@ const App = () => { {showHeader && ( - + )}
diff --git a/src/pages/history/HistoryPage.tsx b/src/pages/history/HistoryPage.tsx index 8349e80a0..dd3df0009 100644 --- a/src/pages/history/HistoryPage.tsx +++ b/src/pages/history/HistoryPage.tsx @@ -9,6 +9,7 @@ import { TableAction, TableColumn, TablePagination, + TableSelection, TableState, useSnackbar, withLoading, @@ -75,7 +76,7 @@ const HistoryPage: React.FC<{ loading: any }> = ({ loading }) => { const [syncRules, setSyncRules] = useState([]); const [syncReport, setSyncReport] = useState(null); const [toDelete, setToDelete] = useState([]); - const [selection, updateSelection] = useState([]); + const [selection, updateSelection] = useState([]); const [response, updateResponse] = useState<{ rows: SynchronizationReport[]; pager: Partial; @@ -102,7 +103,7 @@ const HistoryPage: React.FC<{ loading: any }> = ({ loading }) => { SyncRule.list(d2 as D2, { type }, { paging: false }).then(({ objects }) => setSyncRules(objects) ); - if (!!id) SyncReport.get(d2 as D2, id).then(setSyncReport); + if (id) SyncReport.get(d2 as D2, id).then(setSyncReport); }, [d2, id, type]); useEffect(() => { diff --git a/src/pages/landing/LandingPage.tsx b/src/pages/landing/LandingPage.tsx index de3b9fc4e..e994a3c6c 100644 --- a/src/pages/landing/LandingPage.tsx +++ b/src/pages/landing/LandingPage.tsx @@ -42,92 +42,92 @@ const LandingPage: React.FC = () => { children: MenuCardProps[]; }[] = [ { - title: "Metadata Sync", - key: "metadata", + title: "Aggregated Data Sync", + key: "aggregated", children: [ { name: i18n.t("Manual sync"), description: i18n.t( - "Manually synchronise metadata like data elements, organisation units and program indicators and groups and group sets." + "Manually synchronise aggregated data by selecting the data sets, data elements or their groups and group sets together with the organisation unit, period and category options." ), - listAction: () => history.push("/sync/metadata"), + listAction: () => history.push("/sync/aggregated"), }, { name: i18n.t("Sync rules"), description: i18n.t( - "Create, modify, delete, execute and schedule sync rules for metadata like data elements, organisation units and program indicators and groups and group sets." + "Create, modify, delete, execute and schedule sync rules for aggregated data by selecting the data sets, data elements or their groups and group sets together with the organisation unit, period and category options." ), addAction: showCreateLinks - ? () => history.push("/sync-rules/metadata/new") + ? () => history.push("/sync-rules/aggregated/new") : undefined, - listAction: () => history.push("/sync-rules/metadata"), + listAction: () => history.push("/sync-rules/aggregated"), }, { name: i18n.t("History"), description: i18n.t( - "View and analyse the status and results of the metadata manual syncs and sync rules executions." + "View and analyse the status and results of the aggregated data manual syncs and sync rules executions." ), - listAction: () => history.push("/history/metadata"), + listAction: () => history.push("/history/aggregated"), }, ], }, { - title: "Aggregated Data Sync", - key: "aggregated", + title: "Events Sync", + key: "events", children: [ { name: i18n.t("Manual sync"), description: i18n.t( - "Manually synchronise aggregated data by selecting the data sets, data elements or their groups and group sets together with the organisation unit, period and category options." + "Manually synchronise events by selecting the programs or events together with the organisation unit, period and category options." ), - listAction: () => history.push("/sync/aggregated"), + listAction: () => history.push("/sync/events"), }, { name: i18n.t("Sync rules"), description: i18n.t( - "Create, modify, delete, execute and schedule sync rules for aggregated data by selecting the data sets, data elements or their groups and group sets together with the organisation unit, period and category options." + "Create, modify, delete, execute and schedule sync rules for events by selecting the programs or events together with the organisation unit, period and category options." ), addAction: showCreateLinks - ? () => history.push("/sync-rules/aggregated/new") + ? () => history.push("/sync-rules/events/new") : undefined, - listAction: () => history.push("/sync-rules/aggregated"), + listAction: () => history.push("/sync-rules/events"), }, { name: i18n.t("History"), description: i18n.t( - "View and analyse the status and results of the aggregated data manual syncs and sync rules executions." + "View and analyse the status and results of the event manual syncs and sync rules executions." ), - listAction: () => history.push("/history/aggregated"), + listAction: () => history.push("/history/events"), }, ], }, { - title: "Events Sync", - key: "events", + title: "Metadata Sync", + key: "metadata", children: [ { name: i18n.t("Manual sync"), description: i18n.t( - "Manually synchronise events by selecting the programs or events together with the organisation unit, period and category options." + "Manually synchronise metadata like data elements, organisation units and program indicators and groups and group sets." ), - listAction: () => history.push("/sync/events"), + listAction: () => history.push("/sync/metadata"), }, { name: i18n.t("Sync rules"), description: i18n.t( - "Create, modify, delete, execute and schedule sync rules for events by selecting the programs or events together with the organisation unit, period and category options." + "Create, modify, delete, execute and schedule sync rules for metadata like data elements, organisation units and program indicators and groups and group sets." ), addAction: showCreateLinks - ? () => history.push("/sync-rules/events/new") + ? () => history.push("/sync-rules/metadata/new") : undefined, - listAction: () => history.push("/sync-rules/events"), + listAction: () => history.push("/sync-rules/metadata"), }, { name: i18n.t("History"), description: i18n.t( - "View and analyse the status and results of the event manual syncs and sync rules executions." + "View and analyse the status and results of the metadata manual syncs and sync rules executions." ), - listAction: () => history.push("/history/events"), + listAction: () => history.push("/history/metadata"), }, ], }, diff --git a/src/pages/landing/MenuCard.tsx b/src/pages/landing/MenuCard.tsx index 8ac1af76a..c12b9dc45 100644 --- a/src/pages/landing/MenuCard.tsx +++ b/src/pages/landing/MenuCard.tsx @@ -25,7 +25,7 @@ const useStyles = makeStyles({ width: "230px", }, content: { - height: "85px", + height: "120px", padding: ".5rem 1rem", fontSize: "14px", }, diff --git a/src/pages/sync-on-demand/SyncOnDemandPage.tsx b/src/pages/sync-on-demand/SyncOnDemandPage.tsx index fa403dae0..24867fb18 100644 --- a/src/pages/sync-on-demand/SyncOnDemandPage.tsx +++ b/src/pages/sync-on-demand/SyncOnDemandPage.tsx @@ -10,6 +10,7 @@ import SyncDialog from "../../components/sync-dialog/SyncDialog"; import SyncSummary from "../../components/sync-summary/SyncSummary"; import { AggregatedSync } from "../../logic/sync/aggregated"; import { EventsSync } from "../../logic/sync/events"; +import { SyncronizationClass } from "../../logic/sync/generic"; import { MetadataSync } from "../../logic/sync/metadata"; import { AggregatedDataElementModel, @@ -37,7 +38,7 @@ const config: Record< title: string; models: typeof D2Model[]; childrenKeys: string[] | undefined; - SyncClass: typeof MetadataSync | typeof AggregatedSync | typeof EventsSync; + SyncClass: SyncronizationClass; } > = { metadata: { @@ -89,8 +90,8 @@ const SyncOnDemandPage: React.FC = ({ isDelete, loading } const goBack = () => history.goBack(); - const updateSelection = (selection: string[]) => { - updateSyncRule(syncRule.updateMetadataIds(selection)); + const updateSelection = (selection: string[], exclusion: string[]) => { + updateSyncRule(syncRule.updateMetadataIds(selection).updateExcludedIds(exclusion)); }; const closeSummary = () => { @@ -162,7 +163,8 @@ const SyncOnDemandPage: React.FC = ({ isDelete, loading } } diff --git a/src/pages/sync-rules-list/SyncRulesListPage.jsx b/src/pages/sync-rules-list/SyncRulesListPage.jsx index c9986344c..92c8c970b 100644 --- a/src/pages/sync-rules-list/SyncRulesListPage.jsx +++ b/src/pages/sync-rules-list/SyncRulesListPage.jsx @@ -7,9 +7,7 @@ import { withLoading, withSnackbar, } from "d2-ui-components"; -import FileSaver from "file-saver"; import _ from "lodash"; -import moment from "moment"; import PropTypes from "prop-types"; import React from "react"; import { withRouter } from "react-router-dom"; @@ -30,6 +28,7 @@ import { isAppExecutor, isGlobalAdmin, } from "../../utils/permissions"; +import { requestJSONDownload } from "../../utils/synchronization"; import { getValidationMessages } from "../../utils/validations"; const config = { @@ -163,13 +162,7 @@ class SyncRulesPage extends React.Component { const { SyncClass } = config[syncRule.type]; loading.show(true, "Generating JSON file"); - - const sync = new SyncClass(d2, api, syncRule.toBuilder()); - const payload = await sync.buildPayload(); - - const json = JSON.stringify(payload, null, 4); - const blob = new Blob([json], { type: "application/json" }); - FileSaver.saveAs(blob, `${syncRule.type}-sync-${moment().format("YYYYMMDDHHmm")}.json`); + requestJSONDownload(SyncClass, syncRule, d2, api); loading.reset(); }; diff --git a/src/types/synchronization.d.ts b/src/types/synchronization.d.ts index 6c235c308..7bf4f93b3 100644 --- a/src/types/synchronization.d.ts +++ b/src/types/synchronization.d.ts @@ -1,10 +1,11 @@ import { Ref } from "d2-api"; import SyncReport from "../models/syncReport"; -import { ImportStatus, DataImportParams, MetadataImportParams, MetadataImportStats } from "./d2"; +import { DataImportParams, ImportStatus, MetadataImportParams, MetadataImportStats } from "./d2"; export interface SynchronizationBuilder { targetInstances: string[]; metadataIds: string[]; + excludedIds: string[]; syncRule?: string; syncParams?: MetadataSynchronizationParams; dataParams?: DataSynchronizationParams; diff --git a/src/utils/synchronization.ts b/src/utils/synchronization.ts index 6619f3636..5332b2c90 100644 --- a/src/utils/synchronization.ts +++ b/src/utils/synchronization.ts @@ -2,10 +2,13 @@ import i18n from "@dhis2/d2-i18n"; import axios, { AxiosBasicCredentials } from "axios"; import { D2Api } from "d2-api"; import { isValidUid } from "d2/uid"; +import FileSaver from "file-saver"; import _ from "lodash"; import moment, { Moment } from "moment"; import memoize from "nano-memoize"; +import { SyncronizationClass } from "../logic/sync/generic"; import Instance from "../models/instance"; +import SyncRule from "../models/syncRule"; import { D2, DataImportParams, @@ -19,6 +22,7 @@ import { NestedRules, ProgramEvent, SynchronizationResult, + DataValue, } from "../types/synchronization"; import "../utils/lodash-mixins"; import { cleanModelName, getClassName } from "./d2"; @@ -276,7 +280,7 @@ export async function getAggregatedData( if (dataSet.length === 0 && dataElementGroup.length === 0) return {}; - const orgUnit = _.compact(orgUnitPaths.map(path => _.last(path.split("/")))); + const orgUnit = cleanOrgUnitPaths(orgUnitPaths); const attributeOptionCombo = !allAttributeCategoryOptions ? attributeCategoryOptions : undefined; @@ -294,7 +298,7 @@ export async function getAggregatedData( dataElementGroup, orgUnit, }) - .getData(); + .getData() as Promise<{ dataValues?: DataValue[] }>; } export const getDefaultIds = memoize( @@ -322,6 +326,10 @@ export function cleanObjectDefault(object: ProgramEvent, defaults: string[]) { return _.pickBy(object, value => !defaults.includes(String(value))) as ProgramEvent; } +export function cleanOrgUnitPaths(orgUnitPaths: string[]): string[] { + return _.compact(orgUnitPaths.map(path => _.last(path.split("/")))); +} + export async function getEventsData( api: D2Api, params: DataSynchronizationParams, @@ -333,7 +341,7 @@ export async function getEventsData( if (programs.length === 0) return []; - const orgUnits = _.compact(orgUnitPaths.map(path => _.last(path.split("/")))); + const orgUnits = cleanOrgUnitPaths(orgUnitPaths); const result = []; @@ -439,3 +447,20 @@ export function buildMetadataDictionary(metadataPackage: MetadataPackage) { .keyBy("id") .value(); } + +export async function requestJSONDownload( + SyncClass: SyncronizationClass, + syncRule: SyncRule, + d2: D2, + api: D2Api +) { + const sync = new SyncClass(d2, api, syncRule.toBuilder()); + const payload = await sync.buildPayload(); + + const json = JSON.stringify(payload, null, 4); + const blob = new Blob([json], { type: "application/json" }); + const ruleName = _.kebabCase(_.toLower(syncRule.name)); + const date = moment().format("YYYYMMDDHHmm"); + const fileName = `${ruleName}-${syncRule.type}-sync-${date}.json`; + FileSaver.saveAs(blob, fileName); +} diff --git a/yarn.lock b/yarn.lock index b5bac9f95..f3e6989c5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2168,6 +2168,11 @@ resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== +"@types/file-saver@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.1.tgz#e18eb8b069e442f7b956d313f4fadd3ef887354e" + integrity sha512-g1QUuhYVVAamfCifK7oB7G3aIl4BbOyzDOqVyUfEr4tfBKrXfeH+M+Tg7HKCXSrbzxYdhyCP7z9WbKo0R2hBCw== + "@types/glob@^7.1.1": version "7.1.1" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.1.tgz#aa59a1c6e3fbc421e07ccd31a944c30eba521575" @@ -4827,10 +4832,10 @@ d2-manifest@^1.0.0: minimist "^1.1.0" readline-sync "^1.4.1" -d2-ui-components@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/d2-ui-components/-/d2-ui-components-1.0.1.tgz#d188e1f85069331401919cfa97acf0bcaf17e4b0" - integrity sha512-FA6sPSroc65HzloqNix2VA91GeVBGnQoCyWPOuriFQDOML/XZQFWejuugZktDdsu9w0cE2CUAa/6m+1zuegMcQ== +d2-ui-components@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/d2-ui-components/-/d2-ui-components-1.0.2.tgz#6a2af6ea92c3065bc9a7ea1c69d946f5439d0fb4" + integrity sha512-f/A7iAiQM8llimROn6V8Z7DygNu/ECSyqHHLpVVySympX6H+lL9QwXNqkOvUoZjbdF12YwnJZVxREeG8R0eQkw== dependencies: "@date-io/core" "^1.3.6" "@date-io/moment" "^1.0.2"