From e340ce72bc1fb18f5a866a8dec1c73d2b5b63ed2 Mon Sep 17 00:00:00 2001 From: Greger Teigre Wedel Date: Fri, 1 Dec 2023 16:30:12 +0100 Subject: [PATCH] Fix wrong space names and transformation names (#191) --- .github/workflows/demo.yml | 8 +++---- CHANGELOG.cdf-tk.md | 2 +- cognite_toolkit/_version.py | 2 +- cognite_toolkit/cdf_tk/load.py | 2 +- cognite_toolkit/cdf_tk/templates.py | 18 ++++++++------- ...eld_sync_assets_from_hierarchy_to_apm.yaml | 2 +- ...eld_sync_workorders_to_apm_activities.yaml | 2 +- demo/postproc.py | 22 ++++++++++++------- pyproject.toml | 2 +- .../cdf_infield_location.yaml | 4 ++-- 10 files changed, 36 insertions(+), 28 deletions(-) diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml index c246150e9..d2d9aef40 100644 --- a/.github/workflows/demo.yml +++ b/.github/workflows/demo.yml @@ -38,9 +38,9 @@ jobs: - name: "Build the templates" run: cdf-tk build --build-dir=./build --env=demo ./demo_project # be careful, this works as promised - - name: "Delete existing resources including data" - run: | - cdf-tk clean --env=demo ./build + #- name: "Delete existing resources including data" + # run: | + # cdf-tk clean --env=demo ./build - name: "Verify and create access rights" run: cdf-tk auth verify - name: "Allow some time for data modeling to finish syncing of deletions" @@ -48,7 +48,7 @@ jobs: sleep 30 - name: "Deploy the templates" run: | - cdf-tk deploy --env=demo ./build + cdf-tk deploy --drop --drop-data --env=demo ./build - name: "Run transformations and other post-processing" run: | poetry run python ./demo/postproc.py \ No newline at end of file diff --git a/CHANGELOG.cdf-tk.md b/CHANGELOG.cdf-tk.md index 6d805761e..a1852472c 100644 --- a/CHANGELOG.cdf-tk.md +++ b/CHANGELOG.cdf-tk.md @@ -15,7 +15,7 @@ Changes are grouped as follows: - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. -## [TBD] +## [0.1.0a3] - 2023-12-01 ### Changed diff --git a/cognite_toolkit/_version.py b/cognite_toolkit/_version.py index 496c95b1a..9ee9590b7 100644 --- a/cognite_toolkit/_version.py +++ b/cognite_toolkit/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.1.0a2" +__version__ = "0.1.0a3" __template_version__ = "0.2.0" diff --git a/cognite_toolkit/cdf_tk/load.py b/cognite_toolkit/cdf_tk/load.py index 64cb1862a..a48b86a5b 100644 --- a/cognite_toolkit/cdf_tk/load.py +++ b/cognite_toolkit/cdf_tk/load.py @@ -1210,7 +1210,7 @@ def load_datamodel( ToolGlobals.failed = True return print( - f" Created {len(items.changed)} unchanged {type_}(s) that could have been skipped (--drop specified)." + f" Created {len(items.unchanged)} unchanged {type_}(s) that could have been skipped (--drop specified)." ) if delete_removed and not drop: diff --git a/cognite_toolkit/cdf_tk/templates.py b/cognite_toolkit/cdf_tk/templates.py index 218a3a044..c18d916a6 100644 --- a/cognite_toolkit/cdf_tk/templates.py +++ b/cognite_toolkit/cdf_tk/templates.py @@ -187,10 +187,11 @@ def check_yaml_semantics(parsed: Any, filepath_src: Path, filepath_build: Path, if resource_type == "auth": parts = ext_id.split("_") if len(parts) < 2: - if verbose and ext_id == "applications-configuration": - print( - " [bold green]INFO:[/] the group applications-configuration does not follow the recommended '_' based namespacing because Infield expects this specific name." - ) + if ext_id == "applications-configuration": + if verbose: + print( + " [bold green]INFO:[/] the group applications-configuration does not follow the recommended '_' based namespacing because Infield expects this specific name." + ) else: print( f" [bold yellow]WARNING:[/] the group {filepath_src} has a name [bold]{ext_id}[/] without the recommended '_' based namespacing." @@ -227,10 +228,11 @@ def check_yaml_semantics(parsed: Any, filepath_src: Path, filepath_build: Path, f" [bold yellow]WARNING:[/] the space {filepath_src} has an externalId [bold]{ext_id}[/] without the recommended '_' based namespacing." ) elif parts[0] != "sp": - if verbose and (ext_id == "cognite_app_data" or ext_id == "APM_SourceData" or ext_id == "APM_Config"): - print( - f" [bold green]INFO:[/] the space {ext_id} does not follow the recommended '_' based namespacing because Infield expects this specific name." - ) + if ext_id == "cognite_app_data" or ext_id == "APM_SourceData" or ext_id == "APM_Config": + if verbose: + print( + f" [bold green]INFO:[/] the space {ext_id} does not follow the recommended '_' based namespacing because Infield expects this specific name." + ) else: print( f" [bold yellow]WARNING:[/] the space {filepath_src} has an externalId [bold]{ext_id}[/] without the recommended 'sp_' based prefix." diff --git a/cognite_toolkit/modules/cdf_infield_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.yaml b/cognite_toolkit/modules/cdf_infield_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.yaml index 9e1715663..586e0fd7d 100644 --- a/cognite_toolkit/modules/cdf_infield_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.yaml +++ b/cognite_toolkit/modules/cdf_infield_location/transformations/tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm.yaml @@ -5,7 +5,7 @@ destination: space: cdf_core externalId: Asset version: 'v1' - instanceSpace: 'space_infield_{{default_location}}_source' + instanceSpace: 'space_asset_{{default_location}}_source' type: nodes ignoreNullFields: true shared: true diff --git a/cognite_toolkit/modules/cdf_infield_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.yaml b/cognite_toolkit/modules/cdf_infield_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.yaml index edf766581..efb46ff99 100644 --- a/cognite_toolkit/modules/cdf_infield_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.yaml +++ b/cognite_toolkit/modules/cdf_infield_location/transformations/tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities.yaml @@ -6,7 +6,7 @@ destination: externalId: APM_SourceData version: "1" destinationType: APM_Activity - instanceSpace: 'space_infield_{{default_location}}_source' + instanceSpace: 'space_asset_{{default_location}}_source' type: instances ignoreNullFields: true shared: true diff --git a/demo/postproc.py b/demo/postproc.py index 879b0f611..f82d03323 100755 --- a/demo/postproc.py +++ b/demo/postproc.py @@ -20,18 +20,24 @@ def run() -> None: print("Doing post-processing activities for demo project...") ToolGlobals = CDFToolConfig(client_name="cdf-project-templates") try: - print("Running oid-load-asset-hierarchy...") - ToolGlobals.client.transformations.run(transformation_external_id="oid-load-asset-hierarchy") - print("Running sync_workorders_to_apm_activities...") - ToolGlobals.client.transformations.run(transformation_external_id="sync_workorders_to_apm_activities") + print("Running tr_asset_oid_workmate_asset_hierarchy...") + ToolGlobals.client.transformations.run(transformation_external_id="tr_asset_oid_workmate_asset_hierarchy") + print("Running tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities...") + ToolGlobals.client.transformations.run( + transformation_external_id="tr_workorder_oid_workmate_infield_sync_workorders_to_apm_activities" + ) # Wait until assets are in the hierarchy time.sleep(10.0) - print("Running sync_assets_from_hierarchy_to_apm...") - ToolGlobals.client.transformations.run(transformation_external_id="sync_assets_from_hierarchy_to_apm") + print("Running tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm...") + ToolGlobals.client.transformations.run( + transformation_external_id="tr_asset_oid_workmate_infield_sync_assets_from_hierarchy_to_apm" + ) # Wait until assets are in data models time.sleep(10.0) - print("Running sync_asset_parents_from_hierarchy_to_apm...") - ToolGlobals.client.transformations.run(transformation_external_id="sync_asset_parents_from_hierarchy_to_apm") + print("Running tr_asset_oid_workmate_infield_sync_asset_parents_from_hierarchy_to_apm...") + ToolGlobals.client.transformations.run( + transformation_external_id="tr_asset_oid_workmate_infield_sync_asset_parents_from_hierarchy_to_apm" + ) except Exception as e: log.error(f"Failed to run post-processing activities for demo project:\n{e}") exit(1) diff --git a/pyproject.toml b/pyproject.toml index 57c2fd45c..999a108b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cognite_toolkit" -version = "0.1.0a2" +version = "0.1.0a3" description = "Official Cognite Data Fusion tool for project templates and configuration deployment" authors = ["Cognite AS "] license = "Apache-2" diff --git a/tests/test_approval_modules_snapshots/cdf_infield_location.yaml b/tests/test_approval_modules_snapshots/cdf_infield_location.yaml index 320987647..0c409ed28 100644 --- a/tests/test_approval_modules_snapshots/cdf_infield_location.yaml +++ b/tests/test_approval_modules_snapshots/cdf_infield_location.yaml @@ -409,7 +409,7 @@ Transformation: tokenUri: ${IDP_TOKEN_URL} - dataSetId: 999 destination: - instanceSpace: space_infield_oid_source + instanceSpace: space_asset_oid_source type: nodes view: externalId: Asset @@ -452,7 +452,7 @@ Transformation: externalId: APM_SourceData space: APM_SourceData version: '1' - instanceSpace: space_infield_oid_source + instanceSpace: space_asset_oid_source type: instances destinationOidcCredentials: audience: ${IDP_AUDIENCE}