From b1512a6ec35b5cfe7843642ce70cbf83859c2942 Mon Sep 17 00:00:00 2001 From: Pavel Zubarev Date: Tue, 8 Jun 2021 16:41:01 +0200 Subject: [PATCH 1/3] Extract Assets replication into Flex template --- .github/workflows/pr.yaml | 35 +- build.sh | 8 + legacy/pom.xml | 88 ++++ .../config/job-config-assets-replication.toml | 0 .../config/job-config-events-replication.toml | 0 .../config/job-config-files-replication.toml | 0 .../config/job-config-raw-replication.toml | 0 .../job-config-relationships-replication.toml | 0 .../job-config-sequences-replication.toml | 0 ...job-config-ts-replication-header-only.toml | 0 ...job-config-ts-replication-points-only.toml | 0 .../config/job-config-ts-replication.toml | 0 .../df_templates/bq/cdf-assets-bq_metadata | 0 .../df_templates/bq/cdf-events-bq_metadata | 0 .../df_templates/bq/cdf-files-bq_metadata | 0 .../df_templates/bq/cdf-labels-bq_metadata | 0 .../src}/df_templates/bq/cdf-raw-bq_metadata | 0 .../bq/cdf-relationships-bq_metadata | 0 .../bq/cdf-sequence-columns-bq_metadata | 0 .../bq/cdf-sequence-header-bq_metadata | 0 .../bq/cdf-sequence-rows-bq_metadata | 0 .../df_templates/bq/cdf-ts-header-bq_metadata | 0 .../df_templates/bq/cdf-ts-points-bq_metadata | 0 .../bq/cdf-ts-points-hour-bq_metadata | 0 ...plicate-assets-file-based-api-key_metadata | 0 .../replicate/replicate-assets_metadata | 0 ...plicate-events-file-based-api-key_metadata | 0 .../replicate/replicate-events_metadata | 0 .../replicate/replicate-files_metadata | 0 .../replicate/replicate-raw_metadata | 0 .../replicate-relationships_metadata | 0 .../replicate-ts-file-based-api-key_metadata | 0 .../replicate/replicate-ts_metadata | 0 .../com/cognite/sa/beam/TsPointsToFile.java | 0 .../com/cognite/sa/beam/bq/CdfAssetsBQ.java | 0 .../com/cognite/sa/beam/bq/CdfEventsBQ.java | 0 .../com/cognite/sa/beam/bq/CdfFilesBQ.java | 0 .../com/cognite/sa/beam/bq/CdfLabelsBQ.java | 0 .../java/com/cognite/sa/beam/bq/CdfRawBQ.java | 0 .../sa/beam/bq/CdfRelationshipsBQ.java | 0 .../sa/beam/bq/CdfSequenceColumnsBQ.java | 0 .../sa/beam/bq/CdfSequenceHeaderBQ.java | 0 .../cognite/sa/beam/bq/CdfSequenceRowsBQ.java | 0 .../com/cognite/sa/beam/bq/CdfTsHeaderBQ.java | 0 .../com/cognite/sa/beam/bq/CdfTsPointsBQ.java | 2 - .../cognite/sa/beam/bq/CdfTsPointsHourBQ.java | 0 .../sa/beam/replicate/ReplicateEvents.java | 0 .../sa/beam/replicate/ReplicateFiles.java | 0 .../sa/beam/replicate/ReplicateLabels.java | 0 .../sa/beam/replicate/ReplicateRaw.java | 0 .../replicate/ReplicateRelationships.java | 2 - .../sa/beam/replicate/ReplicateSequences.java | 0 .../sa/beam/replicate/ReplicateTs.java | 0 .../src}/main/resources/logback-test.xml | 0 .../ps1/bq/RunCdfAssetsToBQ-Dataflow.ps1 | 0 .../ps1/bq/RunCdfAssetsToBQ-Direct.ps1 | 0 .../ps1/bq/RunCdfEventsToBQ-Dataflow.ps1 | 0 .../ps1/bq/RunCdfEventsToBQ-Direct.ps1 | 0 .../ps1/bq/RunCdfFilesMetaToBQ-Dataflow.ps1 | 0 .../ps1/bq/RunCdfFilesMetaToBQ-Direct.ps1 | 0 .../ps1/bq/RunCdfLabelsToBQ-Dataflow.ps1 | 0 .../scripts/ps1/bq/RunCdfRawToBQ-Dataflow.ps1 | 0 .../bq/RunCdfRelationshipsToBQ-Dataflow.ps1 | 0 .../bq/RunCdfSequenceColumnsToBQ-Dataflow.ps1 | 0 .../bq/RunCdfSequenceHeaderToBQ-Dataflow.ps1 | 0 .../bq/RunCdfSequenceRowsToBQ-Dataflow.ps1 | 0 .../ps1/bq/RunCdfTsHeaderToBQ-Dataflow.ps1 | 0 .../ps1/bq/RunCdfTsHeaderToBQ-Direct.ps1 | 0 .../bq/RunCdfTsPointsHourToBQ-Dataflow.ps1 | 0 .../ps1/bq/RunCdfTsPointsHourToBQ-Direct.ps1 | 0 .../ps1/bq/RunCdfTsPointsToBQ-Dataflow.ps1 | 0 .../ps1/bq/RunCdfTsPointsToBQ-Direct.ps1 | 0 .../src}/scripts/ps1/bq/bqVariables.ps1 | 0 .../src}/scripts/ps1/deploy/base-config.ps1 | 0 ...Deploy-CdfAssetsToBQ-Dataflow_template.ps1 | 0 ...Deploy-CdfEventsToBQ-Dataflow_template.ps1 | 0 ...loy-CdfFilesMetaToBQ-Dataflow_template.ps1 | 0 ...Deploy-CdfLabelsToBQ-Dataflow_template.ps1 | 0 .../Deploy-CdfRawToBQ-Dataflow_template.ps1 | 0 ...CdfRelationshipsToBQ-Dataflow_template.ps1 | 0 ...fSequenceColumnsToBQ-Dataflow_template.ps1 | 0 ...dfSequenceHeaderToBQ-Dataflow_template.ps1 | 0 ...-CdfSequenceRowsToBQ-Dataflow_template.ps1 | 0 ...ploy-CdfTsHeaderToBQ-Dataflow_template.ps1 | 0 ...-CdfTsPointsHourToBQ-Dataflow_template.ps1 | 0 ...ploy-CdfTsPointsToBQ-Dataflow_template.ps1 | 0 ...ploy-ReplicateAssets-Dataflow_template.ps1 | 10 + ...ploy-ReplicateEvents-Dataflow_template.ps1 | 0 ...eploy-ReplicateFiles-Dataflow_template.ps1 | 0 .../Deploy-ReplicateRaw-Dataflow_template.ps1 | 0 ...plicateRelationships-Dataflow_template.ps1 | 0 .../Deploy-ReplicateTs-Dataflow_template.ps1 | 0 .../replicate/RunReplicateAssets-Dataflow.ps1 | 2 +- .../replicate/RunReplicateEvents-Dataflow.ps1 | 0 .../replicate/RunReplicateFiles-Dataflow.ps1 | 0 .../replicate/RunReplicateRaw-Dataflow.ps1 | 0 .../scripts/ps1/replicate/RunReplicateRaw.ps1 | 0 .../RunReplicateRelationships-Dataflow.ps1 | 0 .../RunReplicateSequences-Dataflow.ps1 | 0 .../ps1/replicate/RunReplicateTs-Dataflow.ps1 | 0 .../ps1/replicate/replicateVariables.ps1 | 0 {src => legacy/src}/scripts/ps1/variables.ps1 | 0 .../replicator/RunReplicateLabels-Dataflow.sh | 0 .../sh/replicator/RunReplicateTs-Dataflow.sh | 0 .../com/cognite/sa/beam/bq/CdfRawBQTest.java | 0 pom.xml | 88 +--- ...ploy-ReplicateAssets-Dataflow_template.ps1 | 10 - templates/pom.xml | 409 ++++++++++++++++++ templates/replicate-assets/pom.xml | 32 ++ .../com/cognite/sa/beam}/ReplicateAssets.java | 120 +++-- .../replicate-assets-command-spec.json | 9 + 111 files changed, 675 insertions(+), 140 deletions(-) create mode 100755 build.sh create mode 100644 legacy/pom.xml rename {src => legacy/src}/config/job-config-assets-replication.toml (100%) rename {src => legacy/src}/config/job-config-events-replication.toml (100%) rename {src => legacy/src}/config/job-config-files-replication.toml (100%) rename {src => legacy/src}/config/job-config-raw-replication.toml (100%) rename {src => legacy/src}/config/job-config-relationships-replication.toml (100%) rename {src => legacy/src}/config/job-config-sequences-replication.toml (100%) rename {src => legacy/src}/config/job-config-ts-replication-header-only.toml (100%) rename {src => legacy/src}/config/job-config-ts-replication-points-only.toml (100%) rename {src => legacy/src}/config/job-config-ts-replication.toml (100%) rename {src => legacy/src}/df_templates/bq/cdf-assets-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-events-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-files-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-labels-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-raw-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-relationships-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-sequence-columns-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-sequence-header-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-sequence-rows-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-ts-header-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-ts-points-bq_metadata (100%) rename {src => legacy/src}/df_templates/bq/cdf-ts-points-hour-bq_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-assets-file-based-api-key_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-assets_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-events-file-based-api-key_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-events_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-files_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-raw_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-relationships_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-ts-file-based-api-key_metadata (100%) rename {src => legacy/src}/df_templates/replicate/replicate-ts_metadata (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/TsPointsToFile.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfAssetsBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfEventsBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfFilesBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfLabelsBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfRawBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfRelationshipsBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfSequenceColumnsBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfSequenceHeaderBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfSequenceRowsBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfTsHeaderBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java (99%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/bq/CdfTsPointsHourBQ.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/replicate/ReplicateEvents.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/replicate/ReplicateFiles.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/replicate/ReplicateLabels.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/replicate/ReplicateRaw.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java (99%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/replicate/ReplicateSequences.java (100%) rename {src => legacy/src}/main/java/com/cognite/sa/beam/replicate/ReplicateTs.java (100%) rename {src => legacy/src}/main/resources/logback-test.xml (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfAssetsToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfAssetsToBQ-Direct.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfEventsToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfEventsToBQ-Direct.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfFilesMetaToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfFilesMetaToBQ-Direct.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfLabelsToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfRawToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfRelationshipsToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfSequenceColumnsToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfSequenceHeaderToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfSequenceRowsToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfTsHeaderToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfTsHeaderToBQ-Direct.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Direct.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfTsPointsToBQ-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/RunCdfTsPointsToBQ-Direct.ps1 (100%) rename {src => legacy/src}/scripts/ps1/bq/bqVariables.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/base-config.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfAssetsToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfEventsToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfFilesMetaToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfLabelsToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfRawToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfRelationshipsToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfSequenceColumnsToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfSequenceHeaderToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfSequenceRowsToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfTsHeaderToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfTsPointsHourToBQ-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/bq/Deploy-CdfTsPointsToBQ-Dataflow_template.ps1 (100%) create mode 100644 legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1 rename {src => legacy/src}/scripts/ps1/deploy/replicate/Deploy-ReplicateEvents-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/replicate/Deploy-ReplicateFiles-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/replicate/Deploy-ReplicateRaw-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/replicate/Deploy-ReplicateRelationships-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/deploy/replicate/Deploy-ReplicateTs-Dataflow_template.ps1 (100%) rename {src => legacy/src}/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1 (72%) rename {src => legacy/src}/scripts/ps1/replicate/RunReplicateEvents-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/replicate/RunReplicateFiles-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/replicate/RunReplicateRaw-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/replicate/RunReplicateRaw.ps1 (100%) rename {src => legacy/src}/scripts/ps1/replicate/RunReplicateRelationships-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/replicate/RunReplicateSequences-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/replicate/RunReplicateTs-Dataflow.ps1 (100%) rename {src => legacy/src}/scripts/ps1/replicate/replicateVariables.ps1 (100%) rename {src => legacy/src}/scripts/ps1/variables.ps1 (100%) rename {src => legacy/src}/scripts/sh/replicator/RunReplicateLabels-Dataflow.sh (100%) rename {src => legacy/src}/scripts/sh/replicator/RunReplicateTs-Dataflow.sh (100%) rename {src => legacy/src}/test/java/com/cognite/sa/beam/bq/CdfRawBQTest.java (100%) delete mode 100644 src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1 create mode 100644 templates/pom.xml create mode 100644 templates/replicate-assets/pom.xml rename {src/main/java/com/cognite/sa/beam/replicate => templates/replicate-assets/src/main/java/com/cognite/sa/beam}/ReplicateAssets.java (83%) create mode 100644 templates/replicate-assets/src/main/resources/replicate-assets-command-spec.json diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ebe45df..60c786d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -2,6 +2,10 @@ name: Java CI on: - push +env: + PROJECT_ID: cognite-inso-sandbox + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + TEMPLATE_MODULE: beam-components jobs: build: @@ -16,4 +20,33 @@ jobs: distribution: 'adopt' - name: Build with Maven - run: mvn --batch-mode --update-snapshots verify \ No newline at end of file + run: mvn --batch-mode --update-snapshots verify + + - id: secrets + name: Access GCP secrets + uses: google-github-actions/get-secretmanager-secrets@v0.2.2 + with: + credentials: ${{ secrets.GCP_SA_KEY }} + secrets: |- + sa-key:${{ env.PROJECT_ID }}/beam-components/latest + + - name: Login with specific service account + uses: GoogleCloudPlatform/github-actions/setup-gcloud@v0.2.1 + with: + project_id: ${{ env.PROJECT_ID }} + service_account_key: ${{ steps.secrets.outputs.sa-key }} + export_default_credentials: true + + - name: Build and push image to Google Container Repository + env: + TARGET_GCR_IMAGE: eu.gcr.io/${{ env.PROJECT_ID }}/${{ env.TEMPLATE_MODULE }} + BASE_CONTAINER_IMAGE: gcr.io/dataflow-templates-base/java11-template-launcher-base + BASE_CONTAINER_IMAGE_VERSION: latest + APP_ROOT: /template/${{ env.TEMPLATE_MODULE }} + COMMAND_SPEC: /template/${{ env.TEMPLATE_MODULE }}/resources/${{ env.TEMPLATE_MODULE }}-command-spec.json + run: mvn clean package -Dimage=${TARGET_GCR_IMAGE} \ + -Dbase-container-image=${BASE_CONTAINER_IMAGE} \ + -Dbase-container-image.version=${BASE_CONTAINER_IMAGE_VERSION} \ + -Dapp-root=${APP_ROOT} \ + -Dcommand-spec=${COMMAND_SPEC} \ + -am -pl ${{ env.TEMPLATE_MODULE }} diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..0f5cb22 --- /dev/null +++ b/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +mvn clean package -Dimage=eu.gcr.io/cognitedata-development/replicate-assets \ + -Dbase-container-image=gcr.io/dataflow-templates-base/java11-template-launcher-base \ + -Dbase-container-image.version=latest \ + -Dapp-root=/template/replicate-assets \ + -Dcommand-spec=/template/replicate-assets/resources/replicate-assets-command-spec.json \ + -am -pl templates/replicate-assets \ No newline at end of file diff --git a/legacy/pom.xml b/legacy/pom.xml new file mode 100644 index 0000000..5cae9fd --- /dev/null +++ b/legacy/pom.xml @@ -0,0 +1,88 @@ + + + 4.0.0 + + com.cognite.data + beam-components-legacy + 0.2.0-SNAPSHOT + + + 11 + 11 + UTF-8 + 11 + + 2.28.0 + 0.9.20-SNAPSHOT + 5.7.0 + 1.2.3 + 3.0.0 + + + + + com.cognite + beam-connector-java + ${beam.connector.version} + + + org.apache.beam + beam-sdks-java-core + ${beam.version} + + + org.apache.beam + beam-sdks-java-io-google-cloud-platform + ${beam.version} + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + runtime + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + compile + + + org.apache.beam + beam-sdks-java-extensions-sql + ${beam.version} + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + ch.qos.logback + logback-classic + ${logback.classic.version} + test + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M4 + + + + + \ No newline at end of file diff --git a/src/config/job-config-assets-replication.toml b/legacy/src/config/job-config-assets-replication.toml similarity index 100% rename from src/config/job-config-assets-replication.toml rename to legacy/src/config/job-config-assets-replication.toml diff --git a/src/config/job-config-events-replication.toml b/legacy/src/config/job-config-events-replication.toml similarity index 100% rename from src/config/job-config-events-replication.toml rename to legacy/src/config/job-config-events-replication.toml diff --git a/src/config/job-config-files-replication.toml b/legacy/src/config/job-config-files-replication.toml similarity index 100% rename from src/config/job-config-files-replication.toml rename to legacy/src/config/job-config-files-replication.toml diff --git a/src/config/job-config-raw-replication.toml b/legacy/src/config/job-config-raw-replication.toml similarity index 100% rename from src/config/job-config-raw-replication.toml rename to legacy/src/config/job-config-raw-replication.toml diff --git a/src/config/job-config-relationships-replication.toml b/legacy/src/config/job-config-relationships-replication.toml similarity index 100% rename from src/config/job-config-relationships-replication.toml rename to legacy/src/config/job-config-relationships-replication.toml diff --git a/src/config/job-config-sequences-replication.toml b/legacy/src/config/job-config-sequences-replication.toml similarity index 100% rename from src/config/job-config-sequences-replication.toml rename to legacy/src/config/job-config-sequences-replication.toml diff --git a/src/config/job-config-ts-replication-header-only.toml b/legacy/src/config/job-config-ts-replication-header-only.toml similarity index 100% rename from src/config/job-config-ts-replication-header-only.toml rename to legacy/src/config/job-config-ts-replication-header-only.toml diff --git a/src/config/job-config-ts-replication-points-only.toml b/legacy/src/config/job-config-ts-replication-points-only.toml similarity index 100% rename from src/config/job-config-ts-replication-points-only.toml rename to legacy/src/config/job-config-ts-replication-points-only.toml diff --git a/src/config/job-config-ts-replication.toml b/legacy/src/config/job-config-ts-replication.toml similarity index 100% rename from src/config/job-config-ts-replication.toml rename to legacy/src/config/job-config-ts-replication.toml diff --git a/src/df_templates/bq/cdf-assets-bq_metadata b/legacy/src/df_templates/bq/cdf-assets-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-assets-bq_metadata rename to legacy/src/df_templates/bq/cdf-assets-bq_metadata diff --git a/src/df_templates/bq/cdf-events-bq_metadata b/legacy/src/df_templates/bq/cdf-events-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-events-bq_metadata rename to legacy/src/df_templates/bq/cdf-events-bq_metadata diff --git a/src/df_templates/bq/cdf-files-bq_metadata b/legacy/src/df_templates/bq/cdf-files-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-files-bq_metadata rename to legacy/src/df_templates/bq/cdf-files-bq_metadata diff --git a/src/df_templates/bq/cdf-labels-bq_metadata b/legacy/src/df_templates/bq/cdf-labels-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-labels-bq_metadata rename to legacy/src/df_templates/bq/cdf-labels-bq_metadata diff --git a/src/df_templates/bq/cdf-raw-bq_metadata b/legacy/src/df_templates/bq/cdf-raw-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-raw-bq_metadata rename to legacy/src/df_templates/bq/cdf-raw-bq_metadata diff --git a/src/df_templates/bq/cdf-relationships-bq_metadata b/legacy/src/df_templates/bq/cdf-relationships-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-relationships-bq_metadata rename to legacy/src/df_templates/bq/cdf-relationships-bq_metadata diff --git a/src/df_templates/bq/cdf-sequence-columns-bq_metadata b/legacy/src/df_templates/bq/cdf-sequence-columns-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-sequence-columns-bq_metadata rename to legacy/src/df_templates/bq/cdf-sequence-columns-bq_metadata diff --git a/src/df_templates/bq/cdf-sequence-header-bq_metadata b/legacy/src/df_templates/bq/cdf-sequence-header-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-sequence-header-bq_metadata rename to legacy/src/df_templates/bq/cdf-sequence-header-bq_metadata diff --git a/src/df_templates/bq/cdf-sequence-rows-bq_metadata b/legacy/src/df_templates/bq/cdf-sequence-rows-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-sequence-rows-bq_metadata rename to legacy/src/df_templates/bq/cdf-sequence-rows-bq_metadata diff --git a/src/df_templates/bq/cdf-ts-header-bq_metadata b/legacy/src/df_templates/bq/cdf-ts-header-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-ts-header-bq_metadata rename to legacy/src/df_templates/bq/cdf-ts-header-bq_metadata diff --git a/src/df_templates/bq/cdf-ts-points-bq_metadata b/legacy/src/df_templates/bq/cdf-ts-points-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-ts-points-bq_metadata rename to legacy/src/df_templates/bq/cdf-ts-points-bq_metadata diff --git a/src/df_templates/bq/cdf-ts-points-hour-bq_metadata b/legacy/src/df_templates/bq/cdf-ts-points-hour-bq_metadata similarity index 100% rename from src/df_templates/bq/cdf-ts-points-hour-bq_metadata rename to legacy/src/df_templates/bq/cdf-ts-points-hour-bq_metadata diff --git a/src/df_templates/replicate/replicate-assets-file-based-api-key_metadata b/legacy/src/df_templates/replicate/replicate-assets-file-based-api-key_metadata similarity index 100% rename from src/df_templates/replicate/replicate-assets-file-based-api-key_metadata rename to legacy/src/df_templates/replicate/replicate-assets-file-based-api-key_metadata diff --git a/src/df_templates/replicate/replicate-assets_metadata b/legacy/src/df_templates/replicate/replicate-assets_metadata similarity index 100% rename from src/df_templates/replicate/replicate-assets_metadata rename to legacy/src/df_templates/replicate/replicate-assets_metadata diff --git a/src/df_templates/replicate/replicate-events-file-based-api-key_metadata b/legacy/src/df_templates/replicate/replicate-events-file-based-api-key_metadata similarity index 100% rename from src/df_templates/replicate/replicate-events-file-based-api-key_metadata rename to legacy/src/df_templates/replicate/replicate-events-file-based-api-key_metadata diff --git a/src/df_templates/replicate/replicate-events_metadata b/legacy/src/df_templates/replicate/replicate-events_metadata similarity index 100% rename from src/df_templates/replicate/replicate-events_metadata rename to legacy/src/df_templates/replicate/replicate-events_metadata diff --git a/src/df_templates/replicate/replicate-files_metadata b/legacy/src/df_templates/replicate/replicate-files_metadata similarity index 100% rename from src/df_templates/replicate/replicate-files_metadata rename to legacy/src/df_templates/replicate/replicate-files_metadata diff --git a/src/df_templates/replicate/replicate-raw_metadata b/legacy/src/df_templates/replicate/replicate-raw_metadata similarity index 100% rename from src/df_templates/replicate/replicate-raw_metadata rename to legacy/src/df_templates/replicate/replicate-raw_metadata diff --git a/src/df_templates/replicate/replicate-relationships_metadata b/legacy/src/df_templates/replicate/replicate-relationships_metadata similarity index 100% rename from src/df_templates/replicate/replicate-relationships_metadata rename to legacy/src/df_templates/replicate/replicate-relationships_metadata diff --git a/src/df_templates/replicate/replicate-ts-file-based-api-key_metadata b/legacy/src/df_templates/replicate/replicate-ts-file-based-api-key_metadata similarity index 100% rename from src/df_templates/replicate/replicate-ts-file-based-api-key_metadata rename to legacy/src/df_templates/replicate/replicate-ts-file-based-api-key_metadata diff --git a/src/df_templates/replicate/replicate-ts_metadata b/legacy/src/df_templates/replicate/replicate-ts_metadata similarity index 100% rename from src/df_templates/replicate/replicate-ts_metadata rename to legacy/src/df_templates/replicate/replicate-ts_metadata diff --git a/src/main/java/com/cognite/sa/beam/TsPointsToFile.java b/legacy/src/main/java/com/cognite/sa/beam/TsPointsToFile.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/TsPointsToFile.java rename to legacy/src/main/java/com/cognite/sa/beam/TsPointsToFile.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfAssetsBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfAssetsBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfAssetsBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfAssetsBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfEventsBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfEventsBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfEventsBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfEventsBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfFilesBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfFilesBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfFilesBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfFilesBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfLabelsBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfLabelsBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfLabelsBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfLabelsBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfRawBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfRawBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfRawBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfRawBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfRelationshipsBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfRelationshipsBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfRelationshipsBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfRelationshipsBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfSequenceColumnsBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfSequenceColumnsBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfSequenceColumnsBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfSequenceColumnsBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfSequenceHeaderBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfSequenceHeaderBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfSequenceHeaderBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfSequenceHeaderBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfSequenceRowsBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfSequenceRowsBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfSequenceRowsBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfSequenceRowsBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfTsHeaderBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfTsHeaderBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfTsHeaderBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfTsHeaderBQ.java diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java similarity index 99% rename from src/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java index 8ad9b11..bdbded6 100644 --- a/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java +++ b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsBQ.java @@ -37,12 +37,10 @@ import org.apache.beam.sdk.options.*; import org.apache.beam.sdk.transforms.*; import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.TypeDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.time.Duration; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; diff --git a/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsHourBQ.java b/legacy/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsHourBQ.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/bq/CdfTsPointsHourBQ.java rename to legacy/src/main/java/com/cognite/sa/beam/bq/CdfTsPointsHourBQ.java diff --git a/src/main/java/com/cognite/sa/beam/replicate/ReplicateEvents.java b/legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateEvents.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/replicate/ReplicateEvents.java rename to legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateEvents.java diff --git a/src/main/java/com/cognite/sa/beam/replicate/ReplicateFiles.java b/legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateFiles.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/replicate/ReplicateFiles.java rename to legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateFiles.java diff --git a/src/main/java/com/cognite/sa/beam/replicate/ReplicateLabels.java b/legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateLabels.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/replicate/ReplicateLabels.java rename to legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateLabels.java diff --git a/src/main/java/com/cognite/sa/beam/replicate/ReplicateRaw.java b/legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateRaw.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/replicate/ReplicateRaw.java rename to legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateRaw.java diff --git a/src/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java b/legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java similarity index 99% rename from src/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java rename to legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java index 4af1ebd..8e0e4aa 100644 --- a/src/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java +++ b/legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateRelationships.java @@ -26,7 +26,6 @@ import com.cognite.client.dto.Relationship; import com.google.common.collect.ImmutableMap; import com.google.protobuf.Int64Value; -import com.google.protobuf.StringValue; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; import org.apache.beam.sdk.metrics.Counter; @@ -41,7 +40,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/src/main/java/com/cognite/sa/beam/replicate/ReplicateSequences.java b/legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateSequences.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/replicate/ReplicateSequences.java rename to legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateSequences.java diff --git a/src/main/java/com/cognite/sa/beam/replicate/ReplicateTs.java b/legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateTs.java similarity index 100% rename from src/main/java/com/cognite/sa/beam/replicate/ReplicateTs.java rename to legacy/src/main/java/com/cognite/sa/beam/replicate/ReplicateTs.java diff --git a/src/main/resources/logback-test.xml b/legacy/src/main/resources/logback-test.xml similarity index 100% rename from src/main/resources/logback-test.xml rename to legacy/src/main/resources/logback-test.xml diff --git a/src/scripts/ps1/bq/RunCdfAssetsToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfAssetsToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfAssetsToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfAssetsToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfAssetsToBQ-Direct.ps1 b/legacy/src/scripts/ps1/bq/RunCdfAssetsToBQ-Direct.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfAssetsToBQ-Direct.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfAssetsToBQ-Direct.ps1 diff --git a/src/scripts/ps1/bq/RunCdfEventsToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfEventsToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfEventsToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfEventsToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfEventsToBQ-Direct.ps1 b/legacy/src/scripts/ps1/bq/RunCdfEventsToBQ-Direct.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfEventsToBQ-Direct.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfEventsToBQ-Direct.ps1 diff --git a/src/scripts/ps1/bq/RunCdfFilesMetaToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfFilesMetaToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfFilesMetaToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfFilesMetaToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfFilesMetaToBQ-Direct.ps1 b/legacy/src/scripts/ps1/bq/RunCdfFilesMetaToBQ-Direct.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfFilesMetaToBQ-Direct.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfFilesMetaToBQ-Direct.ps1 diff --git a/src/scripts/ps1/bq/RunCdfLabelsToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfLabelsToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfLabelsToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfLabelsToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfRawToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfRawToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfRawToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfRawToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfRelationshipsToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfRelationshipsToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfRelationshipsToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfRelationshipsToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfSequenceColumnsToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfSequenceColumnsToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfSequenceColumnsToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfSequenceColumnsToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfSequenceHeaderToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfSequenceHeaderToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfSequenceHeaderToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfSequenceHeaderToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfSequenceRowsToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfSequenceRowsToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfSequenceRowsToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfSequenceRowsToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfTsHeaderToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfTsHeaderToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfTsHeaderToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfTsHeaderToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfTsHeaderToBQ-Direct.ps1 b/legacy/src/scripts/ps1/bq/RunCdfTsHeaderToBQ-Direct.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfTsHeaderToBQ-Direct.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfTsHeaderToBQ-Direct.ps1 diff --git a/src/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Direct.ps1 b/legacy/src/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Direct.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Direct.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfTsPointsHourToBQ-Direct.ps1 diff --git a/src/scripts/ps1/bq/RunCdfTsPointsToBQ-Dataflow.ps1 b/legacy/src/scripts/ps1/bq/RunCdfTsPointsToBQ-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfTsPointsToBQ-Dataflow.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfTsPointsToBQ-Dataflow.ps1 diff --git a/src/scripts/ps1/bq/RunCdfTsPointsToBQ-Direct.ps1 b/legacy/src/scripts/ps1/bq/RunCdfTsPointsToBQ-Direct.ps1 similarity index 100% rename from src/scripts/ps1/bq/RunCdfTsPointsToBQ-Direct.ps1 rename to legacy/src/scripts/ps1/bq/RunCdfTsPointsToBQ-Direct.ps1 diff --git a/src/scripts/ps1/bq/bqVariables.ps1 b/legacy/src/scripts/ps1/bq/bqVariables.ps1 similarity index 100% rename from src/scripts/ps1/bq/bqVariables.ps1 rename to legacy/src/scripts/ps1/bq/bqVariables.ps1 diff --git a/src/scripts/ps1/deploy/base-config.ps1 b/legacy/src/scripts/ps1/deploy/base-config.ps1 similarity index 100% rename from src/scripts/ps1/deploy/base-config.ps1 rename to legacy/src/scripts/ps1/deploy/base-config.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfAssetsToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfAssetsToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfAssetsToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfAssetsToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfEventsToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfEventsToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfEventsToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfEventsToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfFilesMetaToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfFilesMetaToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfFilesMetaToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfFilesMetaToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfLabelsToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfLabelsToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfLabelsToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfLabelsToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfRawToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfRawToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfRawToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfRawToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfRelationshipsToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfRelationshipsToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfRelationshipsToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfRelationshipsToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceColumnsToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceColumnsToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfSequenceColumnsToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceColumnsToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceHeaderToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceHeaderToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfSequenceHeaderToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceHeaderToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceRowsToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceRowsToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfSequenceRowsToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfSequenceRowsToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfTsHeaderToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfTsHeaderToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfTsHeaderToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfTsHeaderToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfTsPointsHourToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfTsPointsHourToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfTsPointsHourToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfTsPointsHourToBQ-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/bq/Deploy-CdfTsPointsToBQ-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/bq/Deploy-CdfTsPointsToBQ-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/bq/Deploy-CdfTsPointsToBQ-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/bq/Deploy-CdfTsPointsToBQ-Dataflow_template.ps1 diff --git a/legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1 new file mode 100644 index 0000000..dadf58d --- /dev/null +++ b/legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1 @@ -0,0 +1,10 @@ +. $PSScriptRoot'\..\..\variables.ps1' +. $PSScriptRoot'\..\base-config.ps1' +$postfix = Get-Postfix +$postfix = '' +# Default variable values are picked up from the above files. +# You can override them with local variable definitions in this file. +# +#------------------------------------------------------------------- + +mvn compile exec:java -D exec.mainClass=com.cognite.sa.beam.ReplicateAssets -D exec.args="--project=$gcpProject --runner=DataFlowRunner --gcpTempLocation=$gcpBucketPrefix$postfix/temp --stagingLocation=$gcpBucketPrefix$postfix/template-stage/replicate/replicate-assets --templateLocation=$gcpBucketPrefix$postfix/template/replicate/replicate-assets --region=europe-west1 --experiments=shuffle_mode=service --numWorkers=1 --maxNumWorkers=2 --experiments=enable_stackdriver_agent_metrics --workerMachineType=e2-standard-2" \ No newline at end of file diff --git a/src/scripts/ps1/deploy/replicate/Deploy-ReplicateEvents-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateEvents-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/replicate/Deploy-ReplicateEvents-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateEvents-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/replicate/Deploy-ReplicateFiles-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateFiles-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/replicate/Deploy-ReplicateFiles-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateFiles-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/replicate/Deploy-ReplicateRaw-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateRaw-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/replicate/Deploy-ReplicateRaw-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateRaw-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/replicate/Deploy-ReplicateRelationships-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateRelationships-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/replicate/Deploy-ReplicateRelationships-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateRelationships-Dataflow_template.ps1 diff --git a/src/scripts/ps1/deploy/replicate/Deploy-ReplicateTs-Dataflow_template.ps1 b/legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateTs-Dataflow_template.ps1 similarity index 100% rename from src/scripts/ps1/deploy/replicate/Deploy-ReplicateTs-Dataflow_template.ps1 rename to legacy/src/scripts/ps1/deploy/replicate/Deploy-ReplicateTs-Dataflow_template.ps1 diff --git a/src/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1 b/legacy/src/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1 similarity index 72% rename from src/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1 rename to legacy/src/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1 index 9afa900..56316f9 100644 --- a/src/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1 +++ b/legacy/src/scripts/ps1/replicate/RunReplicateAssets-Dataflow.ps1 @@ -35,7 +35,7 @@ #--------------------------------------------------------------------- $jobConfigFile = $gcpBucketPrefix + '-test/config/var/job-config-assets-replication.toml' -mvn compile exec:java -D exec.mainClass=com.cognite.sa.beam.replicate.ReplicateAssets -D exec.args="--cdfInputSecret=$cdfInputSecret --cdfInputHost=$cdfInputHost --cdfOutputSecret=$cdfOutputSecret --cdfOutputHost=$cdfOutputHost --jobConfigFile=$jobConfigFile --project=$gcpProject --runner=DataFlowRunner --gcpTempLocation=$gcpBucketPrefix-test/temp --stagingLocation=$gcpBucketPrefix-test/stage/replicate/replicate-assets --region=europe-west1 --experiments=shuffle_mode=service --numWorkers=1 --maxNumWorkers=2 --experiments=enable_stackdriver_agent_metrics,enable_execution_details_collection,use_monitoring_state_manager --workerMachineType=e2-standard-2" +mvn compile exec:java -D exec.mainClass=com.cognite.sa.beam.ReplicateAssets -D exec.args="--cdfInputSecret=$cdfInputSecret --cdfInputHost=$cdfInputHost --cdfOutputSecret=$cdfOutputSecret --cdfOutputHost=$cdfOutputHost --jobConfigFile=$jobConfigFile --project=$gcpProject --runner=DataFlowRunner --gcpTempLocation=$gcpBucketPrefix-test/temp --stagingLocation=$gcpBucketPrefix-test/stage/replicate/replicate-assets --region=europe-west1 --experiments=shuffle_mode=service --numWorkers=1 --maxNumWorkers=2 --experiments=enable_stackdriver_agent_metrics,enable_execution_details_collection,use_monitoring_state_manager --workerMachineType=e2-standard-2" # In order to add profiling: # --profilingAgentConfiguration='{\""APICurated\"" : true}' \ No newline at end of file diff --git a/src/scripts/ps1/replicate/RunReplicateEvents-Dataflow.ps1 b/legacy/src/scripts/ps1/replicate/RunReplicateEvents-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/replicate/RunReplicateEvents-Dataflow.ps1 rename to legacy/src/scripts/ps1/replicate/RunReplicateEvents-Dataflow.ps1 diff --git a/src/scripts/ps1/replicate/RunReplicateFiles-Dataflow.ps1 b/legacy/src/scripts/ps1/replicate/RunReplicateFiles-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/replicate/RunReplicateFiles-Dataflow.ps1 rename to legacy/src/scripts/ps1/replicate/RunReplicateFiles-Dataflow.ps1 diff --git a/src/scripts/ps1/replicate/RunReplicateRaw-Dataflow.ps1 b/legacy/src/scripts/ps1/replicate/RunReplicateRaw-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/replicate/RunReplicateRaw-Dataflow.ps1 rename to legacy/src/scripts/ps1/replicate/RunReplicateRaw-Dataflow.ps1 diff --git a/src/scripts/ps1/replicate/RunReplicateRaw.ps1 b/legacy/src/scripts/ps1/replicate/RunReplicateRaw.ps1 similarity index 100% rename from src/scripts/ps1/replicate/RunReplicateRaw.ps1 rename to legacy/src/scripts/ps1/replicate/RunReplicateRaw.ps1 diff --git a/src/scripts/ps1/replicate/RunReplicateRelationships-Dataflow.ps1 b/legacy/src/scripts/ps1/replicate/RunReplicateRelationships-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/replicate/RunReplicateRelationships-Dataflow.ps1 rename to legacy/src/scripts/ps1/replicate/RunReplicateRelationships-Dataflow.ps1 diff --git a/src/scripts/ps1/replicate/RunReplicateSequences-Dataflow.ps1 b/legacy/src/scripts/ps1/replicate/RunReplicateSequences-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/replicate/RunReplicateSequences-Dataflow.ps1 rename to legacy/src/scripts/ps1/replicate/RunReplicateSequences-Dataflow.ps1 diff --git a/src/scripts/ps1/replicate/RunReplicateTs-Dataflow.ps1 b/legacy/src/scripts/ps1/replicate/RunReplicateTs-Dataflow.ps1 similarity index 100% rename from src/scripts/ps1/replicate/RunReplicateTs-Dataflow.ps1 rename to legacy/src/scripts/ps1/replicate/RunReplicateTs-Dataflow.ps1 diff --git a/src/scripts/ps1/replicate/replicateVariables.ps1 b/legacy/src/scripts/ps1/replicate/replicateVariables.ps1 similarity index 100% rename from src/scripts/ps1/replicate/replicateVariables.ps1 rename to legacy/src/scripts/ps1/replicate/replicateVariables.ps1 diff --git a/src/scripts/ps1/variables.ps1 b/legacy/src/scripts/ps1/variables.ps1 similarity index 100% rename from src/scripts/ps1/variables.ps1 rename to legacy/src/scripts/ps1/variables.ps1 diff --git a/src/scripts/sh/replicator/RunReplicateLabels-Dataflow.sh b/legacy/src/scripts/sh/replicator/RunReplicateLabels-Dataflow.sh similarity index 100% rename from src/scripts/sh/replicator/RunReplicateLabels-Dataflow.sh rename to legacy/src/scripts/sh/replicator/RunReplicateLabels-Dataflow.sh diff --git a/src/scripts/sh/replicator/RunReplicateTs-Dataflow.sh b/legacy/src/scripts/sh/replicator/RunReplicateTs-Dataflow.sh similarity index 100% rename from src/scripts/sh/replicator/RunReplicateTs-Dataflow.sh rename to legacy/src/scripts/sh/replicator/RunReplicateTs-Dataflow.sh diff --git a/src/test/java/com/cognite/sa/beam/bq/CdfRawBQTest.java b/legacy/src/test/java/com/cognite/sa/beam/bq/CdfRawBQTest.java similarity index 100% rename from src/test/java/com/cognite/sa/beam/bq/CdfRawBQTest.java rename to legacy/src/test/java/com/cognite/sa/beam/bq/CdfRawBQTest.java diff --git a/pom.xml b/pom.xml index 7d2f9c7..2a87e58 100644 --- a/pom.xml +++ b/pom.xml @@ -1,88 +1,14 @@ - - 4.0.0 - com.cognite.data beam-components 0.2.0-SNAPSHOT + pom + beam-components - - 11 - 11 - UTF-8 - 11 - - 2.28.0 - 0.9.19 - 5.7.0 - 1.2.3 - 3.0.0 - - - - - com.cognite - beam-connector-java - ${beam.connector.version} - - - org.apache.beam - beam-sdks-java-core - ${beam.version} - - - org.apache.beam - beam-sdks-java-io-google-cloud-platform - ${beam.version} - - - org.apache.beam - beam-runners-direct-java - ${beam.version} - runtime - - - org.apache.beam - beam-runners-google-cloud-dataflow-java - ${beam.version} - compile - - - org.apache.beam - beam-sdks-java-extensions-sql - ${beam.version} - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - ch.qos.logback - logback-classic - ${logback.classic.version} - test - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M4 - - - - + + templates + legacy + \ No newline at end of file diff --git a/src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1 b/src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1 deleted file mode 100644 index 258ca81..0000000 --- a/src/scripts/ps1/deploy/replicate/Deploy-ReplicateAssets-Dataflow_template.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -. $PSScriptRoot'\..\..\variables.ps1' -. $PSScriptRoot'\..\base-config.ps1' -$postfix = Get-Postfix -$postfix = '' -# Default variable values are picked up from the above files. -# You can override them with local variable definitions in this file. -# -#------------------------------------------------------------------- - -mvn compile exec:java -D exec.mainClass=com.cognite.sa.beam.replicate.ReplicateAssets -D exec.args="--project=$gcpProject --runner=DataFlowRunner --gcpTempLocation=$gcpBucketPrefix$postfix/temp --stagingLocation=$gcpBucketPrefix$postfix/template-stage/replicate/replicate-assets --templateLocation=$gcpBucketPrefix$postfix/template/replicate/replicate-assets --region=europe-west1 --experiments=shuffle_mode=service --numWorkers=1 --maxNumWorkers=2 --experiments=enable_stackdriver_agent_metrics --workerMachineType=e2-standard-2" \ No newline at end of file diff --git a/templates/pom.xml b/templates/pom.xml new file mode 100644 index 0000000..c4fb06e --- /dev/null +++ b/templates/pom.xml @@ -0,0 +1,409 @@ + + 4.0.0 + com.cognite.data + templates + 0.2.0-SNAPSHOT + + pom + templates + + + 11 + 11 + UTF-8 + 11 + + 2.28.0 + 0.9.20-SNAPSHOT + 2.6.0 + 5.7.0 + 1.2.3 + 3.0.0 + 3.6.2 + 3.0.1 + 3.0.0-M3 + 3.0.0-M1 + 3.0.2 + 3.0.0 + 3.0.1 + 1.7.0 + 2.21.0 + ${project.basedir}/../../target + + + + + com.cognite + beam-connector-java + ${beam.connector.version} + + + org.apache.beam + beam-sdks-java-core + ${beam.version} + + + org.apache.beam + beam-sdks-java-io-google-cloud-platform + ${beam.version} + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + runtime + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + compile + + + org.apache.beam + beam-sdks-java-extensions-sql + ${beam.version} + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + ch.qos.logback + logback-classic + ${logback.classic.version} + test + + + + + ${mvn-target-dir} + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + + + + + com.google.cloud.tools + jib-maven-plugin + ${jib.maven.version} + + + ${base-container-image}:${base-container-image.version} + + + ${app-root} + INHERIT + + + ${command-spec} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + bundle-and-repackage + package + + shade + + + + ${mvn-target-dir}/dependency-reduced-pom.xml + + true + + + *:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + false + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + analyze-dependencies + + analyze-only + + + + true + false + + + com.google.api.grpc:proto-google-cloud-dlp-v2 + + + org.apache.beam:beam-model-pipeline + + + + + org.slf4j:slf4j-api + + + org.slf4j:slf4j-simple + + + org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core + + + org.apache.beam:beam-runners-google-cloud-dataflow-java + + + org.slf4j:slf4j-simple + + + com.google.code.findbugs:jsr305 + + + org.apache.beam:beam-runners-direct-java + + + org.threeten:threetenbp + + + org.apache.beam:beam-sdks-java-core + + + com.google.auto.value:auto-value-annotations + + + joda-time:joda-time + + + org.apache.beam:beam-vendor-guava-20_0 + + + org.apache.beam:beam-sdks-java-io-google-cloud-platform + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${java.version} + ${java.version} + + true + -parameters + + + + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + bundle-and-repackage + package + + shade + + + true + + + *:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + Teleport for Java, version ${project.version} API + Teleport for Java, version ${project.version} + false + true + true + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven-jar-plugin.version} + + + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven-enforcer-plugin.version} + + + enforce + + enforce + + + + + [10,) + + + + [3.2,) + + + + + + enforce-banned-dependencies + + enforce + + + + + + com.google.guava:guava-jdk5 + com.google.protobuf:protobuf-lite + + + + true + + + + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-6 + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + ${excluded.spanner.tests} + + + + + + + + snapshot + + + apache.snapshots + https://repository.apache.org/content/repositories/snapshots + + + + + + replicate-assets + + \ No newline at end of file diff --git a/templates/replicate-assets/pom.xml b/templates/replicate-assets/pom.xml new file mode 100644 index 0000000..908389c --- /dev/null +++ b/templates/replicate-assets/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + replicate-assets + + + templates + com.cognite.data + 0.2.0-SNAPSHOT + + + + ${mvn-target-dir} + + + com.google.cloud.tools + jib-maven-plugin + + + package + + build + + + + + + + \ No newline at end of file diff --git a/src/main/java/com/cognite/sa/beam/replicate/ReplicateAssets.java b/templates/replicate-assets/src/main/java/com/cognite/sa/beam/ReplicateAssets.java similarity index 83% rename from src/main/java/com/cognite/sa/beam/replicate/ReplicateAssets.java rename to templates/replicate-assets/src/main/java/com/cognite/sa/beam/ReplicateAssets.java index f7ace5f..eb162bf 100644 --- a/src/main/java/com/cognite/sa/beam/replicate/ReplicateAssets.java +++ b/templates/replicate-assets/src/main/java/com/cognite/sa/beam/ReplicateAssets.java @@ -14,16 +14,16 @@ * limitations under the License. */ -package com.cognite.sa.beam.replicate; +package com.cognite.sa.beam; import com.cognite.beam.io.CogniteIO; -import com.cognite.beam.io.config.*; -import com.cognite.client.dto.Asset; -import com.cognite.client.dto.DataSet; import com.cognite.beam.io.RequestParameters; +import com.cognite.beam.io.config.*; import com.cognite.beam.io.transform.toml.ReadTomlStringArray; import com.cognite.beam.io.transform.toml.ReadTomlStringMap; import com.cognite.client.config.UpsertMode; +import com.cognite.client.dto.Asset; +import com.cognite.client.dto.DataSet; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.protobuf.Int64Value; @@ -34,21 +34,27 @@ import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.options.*; import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.values.*; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollectionTuple; +import org.apache.beam.sdk.values.PCollectionView; +import org.apache.beam.sdk.values.TypeDescriptors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; /** * This pipeline reads assets from the specified cdf instance and writes them to a target cdf. - * + *

* The assets will be synchronized via change detection between source and target. That is, the source will be considered * the "master" dataset and the target cdf instance will be updated to match it via upserts and deletes of assets. - * + *

* This job is prepared to be deployed as a template on GCP (Dataflow) + can be executed directly on any runner. */ public class ReplicateAssets { @@ -59,10 +65,10 @@ public class ReplicateAssets { /** * Create target asset: - * - Remove system fields (id, created date, last updated date, parentId) - * - Translate the asset id links to externalParentId - * - Set the key to the rootAssetExternalId - * - Map data set ids. + * - Remove system fields (id, created date, last updated date, parentId) + * - Translate the asset id links to externalParentId + * - Set the key to the rootAssetExternalId + * - Map data set ids. */ private static class PrepareAssets extends DoFn> { PCollectionView> configMap; @@ -137,35 +143,41 @@ public void processElement(@Element Asset input, */ public interface ReplicateAssetsOptions extends PipelineOptions { // The options below can be used for file-based secrets handling. - /* + @Description("The cdf source config file.The name should be in the format of gs:///folder.") @Validation.Required ValueProvider getCdfInputConfigFile(); + void setCdfInputConfigFile(ValueProvider value); @Description("The cdf target config file. The name should be in the format of gs:///folder.") @Validation.Required ValueProvider getCdfOutputConfigFile(); + void setCdfOutputConfigFile(ValueProvider value); -*/ + @Description("The GCP secret holding the source api key. The reference should be ..") @Validation.Required ValueProvider getCdfInputSecret(); + void setCdfInputSecret(ValueProvider value); @Description("The CDF source host name. The default value is https://api.cognitedata.com.") @Default.String("https://api.cognitedata.com") ValueProvider getCdfInputHost(); + void setCdfInputHost(ValueProvider value); @Description("The GCP secret holding the target api key. The reference should be ..") @Validation.Required ValueProvider getCdfOutputSecret(); + void setCdfOutputSecret(ValueProvider value); @Description("The CDF target host name. The default value is https://api.cognitedata.com.") @Default.String("https://api.cognitedata.com") ValueProvider getCdfOutputHost(); + void setCdfOutputHost(ValueProvider value); /** @@ -174,6 +186,7 @@ public interface ReplicateAssetsOptions extends PipelineOptions { @Description("The job config file. The name should be in the format of gs:///folder.") @Validation.Required ValueProvider getJobConfigFile(); + void setJobConfigFile(ValueProvider value); } @@ -186,23 +199,43 @@ public interface ReplicateAssetsOptions extends PipelineOptions { * @param options */ private static PipelineResult runReplicateAssets(ReplicateAssetsOptions options) throws IOException { - /* - Build the project configuration (CDF tenant and api key) based on: - - api key from Secret Manager - - CDF api host - */ - GcpSecretConfig sourceSecret = GcpSecretConfig.of( - ValueProvider.NestedValueProvider.of(options.getCdfInputSecret(), secret -> secret.split("\\.")[0]), - ValueProvider.NestedValueProvider.of(options.getCdfInputSecret(), secret -> secret.split("\\.")[1])); - ProjectConfig sourceConfig = ProjectConfig.create() - .withApiKeyFromGcpSecret(sourceSecret) - .withHost(options.getCdfInputHost()); - GcpSecretConfig targetSecret = GcpSecretConfig.of( - ValueProvider.NestedValueProvider.of(options.getCdfOutputSecret(), secret -> secret.split("\\.")[0]), - ValueProvider.NestedValueProvider.of(options.getCdfOutputSecret(), secret -> secret.split("\\.")[1])); - ProjectConfig targetConfig = ProjectConfig.create() - .withApiKeyFromGcpSecret(targetSecret) - .withHost(options.getCdfOutputHost()); + + ProjectConfig sourceConfig = ProjectConfig.create(); + ProjectConfig targetConfig = ProjectConfig.create(); + + if (options.getCdfInputConfigFile().isAccessible() && + Objects.requireNonNull(options.getCdfInputConfigFile().get()).endsWith("yaml")) { + sourceConfig = sourceConfig.withYaml(Files.readString(Path.of(options.getCdfInputConfigFile().get()))); + } + + if (options.getCdfOutputConfigFile().isAccessible() && + Objects.requireNonNull(options.getCdfOutputConfigFile().get()).endsWith("yaml")) { + targetConfig = targetConfig.withYaml(Files.readString(Path.of(options.getCdfOutputConfigFile().get()))); + } + + // Overwrite host if present explicitly + if (options.getCdfInputHost().isAccessible() && options.getCdfInputHost().get() != null) { + sourceConfig = sourceConfig.withHost(options.getCdfInputHost().get()); + } + + if (options.getCdfOutputHost().isAccessible() && options.getCdfOutputHost().get() != null) { + targetConfig = targetConfig.withHost(options.getCdfOutputHost().get()); + } + + // Overwrite API-KEYs if present explicitly in options + if (options.getCdfInputSecret().isAccessible() && options.getCdfInputSecret().get() != null) { + GcpSecretConfig sourceSecret = GcpSecretConfig.of( + ValueProvider.NestedValueProvider.of(options.getCdfInputSecret(), secret -> secret.split("\\.")[0]), + ValueProvider.NestedValueProvider.of(options.getCdfInputSecret(), secret -> secret.split("\\.")[1])); + sourceConfig = sourceConfig.withApiKeyFromGcpSecret(sourceSecret); + } + + if (options.getCdfOutputSecret().isAccessible() && options.getCdfOutputSecret().get() != null) { + GcpSecretConfig targetSecret = GcpSecretConfig.of( + ValueProvider.NestedValueProvider.of(options.getCdfOutputSecret(), secret -> secret.split("\\.")[0]), + ValueProvider.NestedValueProvider.of(options.getCdfOutputSecret(), secret -> secret.split("\\.")[1])); + targetConfig = targetConfig.withApiKeyFromGcpSecret(targetSecret); + } Pipeline p = Pipeline.create(options); @@ -283,13 +316,13 @@ Build the project configuration (CDF tenant and api key) based on: .apply("Select id + externalId", MapElements .into(TypeDescriptors.kvs(TypeDescriptors.longs(), TypeDescriptors.strings())) .via((DataSet dataSet) -> { - LOG.info("Source dataset - id: {}, extId: {}, name: {}", - dataSet.getId().getValue(), - dataSet.getExternalId().getValue(), - dataSet.getName().getValue()); - return KV.of(dataSet.getId().getValue(), dataSet.getExternalId().getValue()); + LOG.info("Source dataset - id: {}, extId: {}, name: {}", + dataSet.getId().getValue(), + dataSet.getExternalId().getValue(), + dataSet.getName().getValue()); + return KV.of(dataSet.getId().getValue(), dataSet.getExternalId().getValue()); } - )) + )) .apply("Max per key", Max.perKey()) .apply("To map view", View.asMap()); @@ -303,13 +336,13 @@ Build the project configuration (CDF tenant and api key) based on: .apply("Select externalId + id", MapElements .into(TypeDescriptors.kvs(TypeDescriptors.strings(), TypeDescriptors.longs())) .via((DataSet dataSet) -> { - LOG.info("Target dataset - id: {}, extId: {}, name: {}", - dataSet.getId().getValue(), - dataSet.getExternalId().getValue(), - dataSet.getName().getValue()); - return KV.of(dataSet.getExternalId().getValue(), dataSet.getId().getValue()); + LOG.info("Target dataset - id: {}, extId: {}, name: {}", + dataSet.getId().getValue(), + dataSet.getExternalId().getValue(), + dataSet.getName().getValue()); + return KV.of(dataSet.getExternalId().getValue(), dataSet.getId().getValue()); } - )) + )) .apply("Max per key", Max.perKey()) .apply("To map view", View.asMap()); @@ -395,9 +428,10 @@ public void processElement(@Element KV input, /** * Read the pipeline options from args and run the pipeline. + * * @param args */ - public static void main(String[] args) throws IOException{ + public static void main(String[] args) throws IOException { ReplicateAssetsOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(ReplicateAssetsOptions.class); runReplicateAssets(options); } diff --git a/templates/replicate-assets/src/main/resources/replicate-assets-command-spec.json b/templates/replicate-assets/src/main/resources/replicate-assets-command-spec.json new file mode 100644 index 0000000..13022d1 --- /dev/null +++ b/templates/replicate-assets/src/main/resources/replicate-assets-command-spec.json @@ -0,0 +1,9 @@ +{ + "mainClass": "com.cognite.sa.beam.ReplicateAssets", + "classPath": "/template/replicate-assets/*:/template/replicate-assets/libs/*:/template/replicate-assets/classes", + "defaultParameterValues": { + "labels": "{\"goog-dataflow-provided-template-type\":\"flex\",\"goog-dataflow-provided-template-name\":\"replicate-assets\"}", + "autoscalingAlgorithm": "THROUGHPUT_BASED", + "maxNumWorkers": "10" + } +} \ No newline at end of file From 3b06333b6990481a0bfdc6cfb02b30d223c3a0ef Mon Sep 17 00:00:00 2001 From: Pavel Zubarev Date: Tue, 8 Jun 2021 16:47:23 +0200 Subject: [PATCH 2/3] fix gh action --- .github/workflows/pr.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 60c786d..380282d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -5,10 +5,14 @@ on: env: PROJECT_ID: cognite-inso-sandbox GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - TEMPLATE_MODULE: beam-components jobs: build: + strategy: + matrix: + module: + - replicate-assets + runs-on: ubuntu-latest steps: @@ -39,14 +43,13 @@ jobs: - name: Build and push image to Google Container Repository env: - TARGET_GCR_IMAGE: eu.gcr.io/${{ env.PROJECT_ID }}/${{ env.TEMPLATE_MODULE }} + TARGET_GCR_IMAGE: eu.gcr.io/${{ env.PROJECT_ID }}/${{ matrix.module }} BASE_CONTAINER_IMAGE: gcr.io/dataflow-templates-base/java11-template-launcher-base BASE_CONTAINER_IMAGE_VERSION: latest - APP_ROOT: /template/${{ env.TEMPLATE_MODULE }} - COMMAND_SPEC: /template/${{ env.TEMPLATE_MODULE }}/resources/${{ env.TEMPLATE_MODULE }}-command-spec.json + APP_ROOT: /template/${{ matrix.module }} run: mvn clean package -Dimage=${TARGET_GCR_IMAGE} \ -Dbase-container-image=${BASE_CONTAINER_IMAGE} \ -Dbase-container-image.version=${BASE_CONTAINER_IMAGE_VERSION} \ -Dapp-root=${APP_ROOT} \ - -Dcommand-spec=${COMMAND_SPEC} \ - -am -pl ${{ env.TEMPLATE_MODULE }} + -Dcommand-spec=${APP_ROOT}/resources/${{ matrix.module }}-command-spec.json \ + -am -pl templates/${{ matrix.module }} From d89a440dd627f5bf3e41497f38dd600f6b96c698 Mon Sep 17 00:00:00 2001 From: Pavel Zubarev Date: Tue, 8 Jun 2021 16:49:13 +0200 Subject: [PATCH 3/3] simplify action --- .github/workflows/pr.yaml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 380282d..73d791e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -4,8 +4,7 @@ on: - push env: PROJECT_ID: cognite-inso-sandbox - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - + jobs: build: strategy: @@ -26,19 +25,11 @@ jobs: - name: Build with Maven run: mvn --batch-mode --update-snapshots verify - - id: secrets - name: Access GCP secrets - uses: google-github-actions/get-secretmanager-secrets@v0.2.2 - with: - credentials: ${{ secrets.GCP_SA_KEY }} - secrets: |- - sa-key:${{ env.PROJECT_ID }}/beam-components/latest - - name: Login with specific service account uses: GoogleCloudPlatform/github-actions/setup-gcloud@v0.2.1 with: project_id: ${{ env.PROJECT_ID }} - service_account_key: ${{ steps.secrets.outputs.sa-key }} + service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - name: Build and push image to Google Container Repository