From ef30e9180ecfd87076fbbe9f5e7b9c591c2a82dc Mon Sep 17 00:00:00 2001 From: Aloys Zhang Date: Mon, 28 Oct 2024 14:37:47 +0800 Subject: [PATCH] [INLONG-11439][CI] Support parallel build --- .github/workflows/ci_build.yml | 3 ++- .github/workflows/ci_ut.yml | 4 ++-- .github/workflows/ci_ut_flink13.yml | 4 ++-- .github/workflows/ci_ut_flink15.yml | 4 ++-- .github/workflows/ci_ut_flink18.yml | 4 ++-- .github/workflows/codeql_analysis.yml | 2 +- inlong-audit/audit-service/pom.xml | 2 +- inlong-audit/audit-store/pom.xml | 2 +- inlong-sdk/dataproxy-sdk/pom.xml | 2 +- inlong-sort/pom.xml | 2 +- inlong-sort/sort-dist/pom.xml | 1 + inlong-sort/sort-flink/base/pom.xml | 1 + .../sort-flink/sort-flink-v1.13/sort-connectors/doris/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/elasticsearch-6/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/elasticsearch-7/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/filesystem/pom.xml | 2 ++ .../sort-flink/sort-flink-v1.13/sort-connectors/hbase/pom.xml | 1 + .../sort-flink/sort-flink-v1.13/sort-connectors/hive/pom.xml | 1 + .../sort-flink/sort-flink-v1.13/sort-connectors/hudi/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/iceberg/pom.xml | 1 + .../sort-flink/sort-flink-v1.13/sort-connectors/jdbc/pom.xml | 1 + .../sort-flink/sort-flink-v1.13/sort-connectors/kafka/pom.xml | 1 + .../sort-flink/sort-flink-v1.13/sort-connectors/kudu/pom.xml | 4 +--- .../sort-flink-v1.13/sort-connectors/mongodb-cdc/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/mysql-cdc/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/oracle-cdc/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/postgres-cdc/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/pulsar/pom.xml | 1 + .../sort-flink/sort-flink-v1.13/sort-connectors/redis/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/sqlserver-cdc/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/starrocks/pom.xml | 1 + .../sort-flink-v1.13/sort-connectors/tubemq/pom.xml | 1 + .../sort-connectors/elasticsearch-base/pom.xml | 1 + .../sort-flink/sort-flink-v1.15/sort-connectors/hbase/pom.xml | 1 + .../sort-flink/sort-flink-v1.15/sort-connectors/hudi/pom.xml | 1 + .../sort-flink-v1.15/sort-connectors/iceberg/pom.xml | 2 +- .../sort-flink/sort-flink-v1.15/sort-connectors/jdbc/pom.xml | 1 + .../sort-flink/sort-flink-v1.15/sort-connectors/kafka/pom.xml | 1 + .../sort-flink-v1.15/sort-connectors/mongodb-cdc/pom.xml | 1 + .../sort-flink-v1.15/sort-connectors/mysql-cdc/pom.xml | 1 + .../sort-flink-v1.15/sort-connectors/postgres-cdc/pom.xml | 1 + .../sort-flink-v1.15/sort-connectors/pulsar/pom.xml | 1 + .../sort-flink/sort-flink-v1.15/sort-connectors/redis/pom.xml | 1 + .../sort-flink-v1.15/sort-connectors/sqlserver-cdc/pom.xml | 1 + .../sort-flink-v1.15/sort-connectors/starrocks/pom.xml | 3 +-- .../sort-flink-v1.15/sort-connectors/tubemq/pom.xml | 3 +-- .../sort-connectors/elasticsearch-base/pom.xml | 1 + .../sort-flink-v1.18/sort-connectors/elasticsearch6/pom.xml | 1 + .../sort-flink-v1.18/sort-connectors/elasticsearch7/pom.xml | 1 + .../sort-flink/sort-flink-v1.18/sort-connectors/jdbc/pom.xml | 1 + .../sort-flink-v1.18/sort-connectors/pulsar/pom.xml | 1 + 51 files changed, 57 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml index 9c703d4517..ae6d3aefc4 100644 --- a/.github/workflows/ci_build.yml +++ b/.github/workflows/ci_build.yml @@ -107,7 +107,8 @@ jobs: - name: Build with Maven run: | - mvn --batch-mode --update-snapshots -e -V clean install -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + mvn -B -U -ntp -e -V -T 1C clean install -pl '!inlong-distribution' -am -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + mvn install -pl inlong-distribution -am -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true env: CI: false diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index 21d56a8b93..2d9c157893 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -115,12 +115,12 @@ jobs: sudo swapon /swapfile - name: Build with Maven - run: mvn --batch-mode --update-snapshots -e -V clean install -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + run: mvn -B -U -ntp -e -V clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -pl '!inlong-distribution' -am -T 1C -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false - name: Unit test with Maven - run: mvn --batch-mode --update-snapshots -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13,!:sort-end-to-end-tests-v1.18 + run: mvn -B -U -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13,!:sort-end-to-end-tests-v1.18 env: CI: false diff --git a/.github/workflows/ci_ut_flink13.yml b/.github/workflows/ci_ut_flink13.yml index 3889e27d91..9084e34a3a 100644 --- a/.github/workflows/ci_ut_flink13.yml +++ b/.github/workflows/ci_ut_flink13.yml @@ -84,12 +84,12 @@ jobs: restore-keys: ${{ runner.os }}-inlong-flink13 - name: Build for Flink 1.13 with Maven - run: mvn --update-snapshots -e -V clean install -U -pl :sort-core,:sort-end-to-end-tests-v1.13 -am -Pv1.13 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + run: mvn -B -U -ntp -e -V -T 1C clean install -pl :sort-core,:sort-end-to-end-tests-v1.13 -am -Pv1.13 -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false - name: Unit test for Flink 1.13 with Maven - run: mvn --update-snapshots -e -V verify -pl :sort-core,:sort-end-to-end-tests-v1.13 -am -Pv1.13 + run: mvn -U -e -V verify -pl :sort-core,:sort-end-to-end-tests-v1.13 -am -Pv1.13 env: CI: false diff --git a/.github/workflows/ci_ut_flink15.yml b/.github/workflows/ci_ut_flink15.yml index d5200b6475..53b88957e0 100644 --- a/.github/workflows/ci_ut_flink15.yml +++ b/.github/workflows/ci_ut_flink15.yml @@ -84,12 +84,12 @@ jobs: restore-keys: ${{ runner.os }}-inlong-flink15 - name: Build for Flink 1.15 with Maven - run: mvn --update-snapshots -e -V clean install -U -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + run: mvn -B -U -ntp -e -V -T 1C clean install -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false - name: Unit test for Flink 1.15 with Maven - run: mvn --update-snapshots -e -V verify -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 + run: mvn -U -e -V verify -pl :sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 env: CI: false diff --git a/.github/workflows/ci_ut_flink18.yml b/.github/workflows/ci_ut_flink18.yml index bec6330283..d86ea53bfc 100644 --- a/.github/workflows/ci_ut_flink18.yml +++ b/.github/workflows/ci_ut_flink18.yml @@ -84,12 +84,12 @@ jobs: restore-keys: ${{ runner.os }}-inlong-flink18 - name: Build for Flink 1.18 with Maven - run: mvn --update-snapshots -e -V clean install -U -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + run: mvn -B -U -ntp -e -V -T 1C clean install -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false - name: Unit test for Flink 1.18 with Maven - run: mvn --update-snapshots -e -V verify -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 + run: mvn -U -e -V verify -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 env: CI: false diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml index d9c8b45380..a303eeb780 100644 --- a/.github/workflows/codeql_analysis.yml +++ b/.github/workflows/codeql_analysis.yml @@ -92,7 +92,7 @@ jobs: - name: Build with Maven run: | - mvn --batch-mode --update-snapshots -e -V clean install -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 + mvn -B -U -ntp -e -V -T 1C clean install -pl '!inlong-distribution' -am -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 env: CI: false diff --git a/inlong-audit/audit-service/pom.xml b/inlong-audit/audit-service/pom.xml index e73e77b70a..f04dcfe680 100644 --- a/inlong-audit/audit-service/pom.xml +++ b/inlong-audit/audit-service/pom.xml @@ -228,7 +228,7 @@ maven-shade-plugin true - true + false *:* diff --git a/inlong-audit/audit-store/pom.xml b/inlong-audit/audit-store/pom.xml index 60d69a3e19..74e46ff317 100644 --- a/inlong-audit/audit-store/pom.xml +++ b/inlong-audit/audit-store/pom.xml @@ -366,7 +366,7 @@ maven-shade-plugin true - true + false *:* diff --git a/inlong-sdk/dataproxy-sdk/pom.xml b/inlong-sdk/dataproxy-sdk/pom.xml index 8cccaa7d6e..92d8acc89a 100644 --- a/inlong-sdk/dataproxy-sdk/pom.xml +++ b/inlong-sdk/dataproxy-sdk/pom.xml @@ -126,6 +126,7 @@ org.apache.maven.plugins maven-shade-plugin + false org.apache.inlong:* @@ -153,7 +154,6 @@ org.apache.inlong.dataproxy.shaded.org.apache.thrift - false true diff --git a/inlong-sort/pom.xml b/inlong-sort/pom.xml index 8ebd3e2244..9a01a92cdd 100644 --- a/inlong-sort/pom.xml +++ b/inlong-sort/pom.xml @@ -114,7 +114,7 @@ package false - true + false ${project.basedir}/target/dependency-reduced-pom.xml true diff --git a/inlong-sort/sort-dist/pom.xml b/inlong-sort/sort-dist/pom.xml index 82b3ed9d0d..6c48ff8f91 100644 --- a/inlong-sort/sort-dist/pom.xml +++ b/inlong-sort/sort-dist/pom.xml @@ -74,6 +74,7 @@ package + false false ${project.artifactId}-${project.version} diff --git a/inlong-sort/sort-flink/base/pom.xml b/inlong-sort/sort-flink/base/pom.xml index e66ce3076e..3fdc18e474 100644 --- a/inlong-sort/sort-flink/base/pom.xml +++ b/inlong-sort/sort-flink/base/pom.xml @@ -120,6 +120,7 @@ package + false io.opentelemetry* diff --git a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/doris/pom.xml b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/doris/pom.xml index a156546f9a..ddd2123663 100644 --- a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/doris/pom.xml +++ b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/doris/pom.xml @@ -75,6 +75,7 @@ package + false org.apache.inlong:sort-connector-* diff --git a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/elasticsearch-6/pom.xml b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/elasticsearch-6/pom.xml index cf255b394b..a4da428d45 100644 --- a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/elasticsearch-6/pom.xml +++ b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/elasticsearch-6/pom.xml @@ -168,6 +168,7 @@ package + false false diff --git a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/elasticsearch-7/pom.xml b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/elasticsearch-7/pom.xml index 86275fce54..543d5e75cf 100644 --- a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/elasticsearch-7/pom.xml +++ b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/elasticsearch-7/pom.xml @@ -157,6 +157,7 @@ package + false false diff --git a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/filesystem/pom.xml b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/filesystem/pom.xml index d5ed95070f..28aad16c69 100644 --- a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/filesystem/pom.xml +++ b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/filesystem/pom.xml @@ -45,6 +45,7 @@ org.apache.maven.plugins maven-shade-plugin + shade-flink @@ -53,6 +54,7 @@ package + false org.apache.inlong:* diff --git a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/hbase/pom.xml b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/hbase/pom.xml index 4231196d95..7cc096a93b 100644 --- a/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/hbase/pom.xml +++ b/inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/hbase/pom.xml @@ -79,6 +79,7 @@ package + false