From 63e2a3eb6956148cf12088cd521367f3c2b1dda5 Mon Sep 17 00:00:00 2001 From: Clay Downs <89109232+downsrob@users.noreply.github.com> Date: Mon, 4 Oct 2021 08:44:10 -0700 Subject: [PATCH 1/4] Updates alerting version to 1.2 (#192) * Updates alerting version to 1.2 * Adds snapshot repo to the repository file Signed-off-by: Clay Downs --- .../workflows/multi-node-test-workflow.yml | 22 +--------------- .github/workflows/test-workflow.yml | 26 ++----------------- build-tools/repositories.gradle | 1 + build.gradle | 3 ++- 4 files changed, 6 insertions(+), 46 deletions(-) diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index 6ba35e52f..3511f7d9a 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -23,26 +23,6 @@ jobs: uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - # This step adds dependency, OpenSearch - - name: Checkout OpenSearch - uses: actions/checkout@v2 - with: - repository: 'opensearch-project/OpenSearch' - path: OpenSearch - ref: '1.1' - - name: Build OpenSearch - working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal - # This step adds dependency, common-utils - - name: Checkout common-utils - uses: actions/checkout@v2 - with: - repository: 'opensearch-project/common-utils' - path: common-utils - ref: 'main' - - name: Build common-utils - working-directory: ./common-utils - run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT # This step uses the checkout Github action: https://github.com/actions/checkout - name: Checkout Branch uses: actions/checkout@v2 @@ -52,7 +32,7 @@ jobs: with: java-version: 14 - name: Run integration tests with multi node config - run: ./gradlew integTest -PnumNodes=3 -Dopensearch.version=1.1.0-SNAPSHOT + run: ./gradlew integTest -PnumNodes=3 -Dopensearch.version=1.2.0-SNAPSHOT - name: Pull and Run Docker run: | plugin=`ls alerting/build/distributions/*.zip` diff --git a/.github/workflows/test-workflow.yml b/.github/workflows/test-workflow.yml index 3c68b05ef..30de39ef5 100644 --- a/.github/workflows/test-workflow.yml +++ b/.github/workflows/test-workflow.yml @@ -29,30 +29,8 @@ jobs: with: java-version: ${{ matrix.java }} - # dependencies: OpenSearch - - name: Checkout OpenSearch - uses: actions/checkout@v2 - with: - repository: 'opensearch-project/OpenSearch' - path: OpenSearch - ref: '1.1' - - name: Build OpenSearch - working-directory: ./OpenSearch - run: ./gradlew publishToMavenLocal - - # dependencies: common-utils - - name: Checkout common-utils - uses: actions/checkout@v2 - with: - repository: 'opensearch-project/common-utils' - path: common-utils - ref: 'main' - - name: Build common-utils - working-directory: ./common-utils - run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT - - name: Build and run with Gradle - run: ./gradlew build -Dopensearch.version=1.1.0-SNAPSHOT + run: ./gradlew build -Dopensearch.version=1.2.0-SNAPSHOT # - name: Create Artifact Path # run: | @@ -71,4 +49,4 @@ jobs: # path: alerting-artifacts # Publish to local maven - name: Publish to Maven Local - run: ./gradlew publishToMavenLocal -Dopensearch.version=1.1.0-SNAPSHOT + run: ./gradlew publishToMavenLocal -Dopensearch.version=1.2.0-SNAPSHOT diff --git a/build-tools/repositories.gradle b/build-tools/repositories.gradle index 2421552ca..1d4a64244 100644 --- a/build-tools/repositories.gradle +++ b/build-tools/repositories.gradle @@ -28,4 +28,5 @@ repositories { mavenLocal() mavenCentral() jcenter() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } } \ No newline at end of file diff --git a/build.gradle b/build.gradle index c92b6430d..2e23b8ed9 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ buildscript { apply from: 'build-tools/repositories.gradle' ext { - opensearch_version = System.getProperty("opensearch.version", "1.1.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "1.2.0-SNAPSHOT") // 1.0.0 -> 1.0.0.0, and 1.0.0-SNAPSHOT -> 1.0.0.0-SNAPSHOT opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2') common_utils_version = System.getProperty("common_utils.version", opensearch_build) @@ -40,6 +40,7 @@ buildscript { mavenCentral() maven { url "https://plugins.gradle.org/m2/" } jcenter() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } } dependencies { classpath "org.opensearch.gradle:build-tools:${opensearch_version}" From 948aa55d6d254f4cf877e9a437a3125c5197a03a Mon Sep 17 00:00:00 2001 From: Abbas Hussain Date: Mon, 4 Oct 2021 09:33:31 -0700 Subject: [PATCH 2/4] Update build to use public Maven repo (#184) Signed-off-by: Abbas Hussain --- build-tools/repositories.gradle | 1 + build.gradle | 1 + 2 files changed, 2 insertions(+) diff --git a/build-tools/repositories.gradle b/build-tools/repositories.gradle index 1d4a64244..3a3957d4d 100644 --- a/build-tools/repositories.gradle +++ b/build-tools/repositories.gradle @@ -26,6 +26,7 @@ repositories { mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() jcenter() maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } diff --git a/build.gradle b/build.gradle index 2e23b8ed9..49a17decc 100644 --- a/build.gradle +++ b/build.gradle @@ -37,6 +37,7 @@ buildscript { repositories { mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() maven { url "https://plugins.gradle.org/m2/" } jcenter() From dbebe95c6eb487d00a45b077ad051d2fe851622e Mon Sep 17 00:00:00 2001 From: Sriram <59816283+skkosuri-amzn@users.noreply.github.com> Date: Wed, 6 Oct 2021 12:33:37 -0700 Subject: [PATCH 3/4] Add valid search filters. (#194) --- .../resthandler/RestSearchMonitorAction.kt | 9 ++++++++- .../alerting/resthandler/MonitorRestApiIT.kt | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorAction.kt index d19cd6560..a7b167af9 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandler/RestSearchMonitorAction.kt @@ -31,6 +31,7 @@ import org.opensearch.action.search.SearchResponse import org.opensearch.alerting.AlertingPlugin import org.opensearch.alerting.action.SearchMonitorAction import org.opensearch.alerting.action.SearchMonitorRequest +import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_INDEX_PATTERN import org.opensearch.alerting.core.model.ScheduledJob import org.opensearch.alerting.core.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX import org.opensearch.alerting.model.Monitor @@ -108,12 +109,18 @@ class RestSearchMonitorAction( log.debug("${request.method()} ${AlertingPlugin.MONITOR_BASE_URI}/_search") val index = request.param("index", SCHEDULED_JOBS_INDEX) + if (index != SCHEDULED_JOBS_INDEX && index != ALL_INDEX_PATTERN) { + throw IllegalArgumentException("Invalid index name.") + } + val searchSourceBuilder = SearchSourceBuilder() searchSourceBuilder.parseXContent(request.contentOrSourceParamParser()) searchSourceBuilder.fetchSource(context(request)) val queryBuilder = QueryBuilders.boolQuery().must(searchSourceBuilder.query()) - queryBuilder.filter(QueryBuilders.existsQuery(Monitor.MONITOR_TYPE)) + if (index == SCHEDULED_JOBS_INDEX) { + queryBuilder.filter(QueryBuilders.existsQuery(Monitor.MONITOR_TYPE)) + } searchSourceBuilder.query(queryBuilder) .seqNoAndPrimaryTerm(true) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt index b1a1cd916..730b3b241 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorRestApiIT.kt @@ -942,4 +942,22 @@ class MonitorRestApiIT : AlertingRestTestCase() { val monitorHit = hit["_source"] as Map assertEquals("Type is not monitor", monitorHit[Monitor.TYPE_FIELD], "monitor") } + + @Throws(Exception::class) + fun `test search monitor with alerting indices only`() { + // 1. search - must return error as invalid index is passed + val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString() + val params: MutableMap = HashMap() + params["index"] = "data-logs" + try { + client().makeRequest( + "GET", + "$ALERTING_BASE_URI/_search", + params, + NStringEntity(search, ContentType.APPLICATION_JSON) + ) + } catch (e: ResponseException) { + assertEquals("Unexpected status", RestStatus.BAD_REQUEST, e.response.restStatus()) + } + } } From 7959450a7bea855dd7d3c280b106ac8de116b146 Mon Sep 17 00:00:00 2001 From: "Daniel Doubrovkine (dB.)" Date: Wed, 6 Oct 2021 20:02:02 -0400 Subject: [PATCH 4/4] Publish notification JARs checksums. (#196) * Publish notification JARs checksums. Signed-off-by: dblock * Remove sonatype staging. Signed-off-by: dblock --- .github/workflows/push-notification-jar.yml | 44 ------------ notification/build.gradle | 25 +++---- scripts/build.sh | 75 +++++++++++++++++++++ 3 files changed, 84 insertions(+), 60 deletions(-) delete mode 100644 .github/workflows/push-notification-jar.yml create mode 100755 scripts/build.sh diff --git a/.github/workflows/push-notification-jar.yml b/.github/workflows/push-notification-jar.yml deleted file mode 100644 index 3b76e5854..000000000 --- a/.github/workflows/push-notification-jar.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Upload Notification Jar to Maven - -on: - push: - tags: - - v* -jobs: - upload-notification-jar: - runs-on: [ubuntu-16.04] - name: Upload Notification Jar to Maven - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - - name: Configure AWS CLI - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Setup Java - uses: actions/setup-java@v1 - with: - java-version: '14' - - - name: Upload Notification Jar to Maven - env: - passphrase: ${{ secrets.PASSPHRASE }} - run: | - cd .. - export JAVA14_HOME=$JAVA_HOME - aws s3 cp s3://opendistro-docs/github-actions/pgp-public-key . - aws s3 cp s3://opendistro-docs/github-actions/pgp-private-key . - - gpg --import pgp-public-key - gpg --allow-secret-key-import --import pgp-private-key - - mkdir /home/runner/.gradle - aws s3 cp s3://opendistro-docs/github-actions/gradle.properties /home/runner/.gradle/ - - cd alerting/notification - - ../gradlew publishShadowPublicationToSonatype-stagingRepository -Dcompiler.java=14 -Dbuild.snapshot=false -Djavax.net.ssl.trustStore=$JAVA_HOME/lib/security/cacerts diff --git a/notification/build.gradle b/notification/build.gradle index 5fb42ccaf..62c2acae7 100644 --- a/notification/build.gradle +++ b/notification/build.gradle @@ -57,6 +57,15 @@ task javadocJar(type: Jar) { from javadoc.destinationDir } +tasks.withType(Jar) { task -> + task.doLast { + ant.checksum algorithm: 'md5', file: it.archivePath + ant.checksum algorithm: 'sha1', file: it.archivePath + ant.checksum algorithm: 'sha-256', file: it.archivePath, fileext: '.sha256' + ant.checksum algorithm: 'sha-512', file: it.archivePath, fileext: '.sha512' + } +} + publishing { publications { shadow(MavenPublication) { @@ -94,23 +103,7 @@ publishing { } } - repositories { - maven { - name = "sonatype-staging" - url "https://aws.oss.sonatype.org/service/local/staging/deploy/maven2" - credentials { - username project.hasProperty('ossrhUsername') ? project.property('ossrhUsername') : '' - password project.hasProperty('ossrhPassword') ? project.property('ossrhPassword') : '' - } - } - } - // TODO - enabled debug logging for the time being, remove this eventually gradle.startParameter.setShowStacktrace(ShowStacktrace.ALWAYS) gradle.startParameter.setLogLevel(LogLevel.DEBUG) - - signing { - required { gradle.taskGraph.hasTask("publishShadowPublicationToSonatype-stagingRepository") } - sign publishing.publications.shadow - } } diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 000000000..19bdef2bd --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +set -ex + +function usage() { + echo "Usage: $0 [args]" + echo "" + echo "Arguments:" + echo -e "-v VERSION\t[Required] OpenSearch version." + echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'." + echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored." + echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'." + echo -e "-h help" +} + +while getopts ":h:v:s:o:a:" arg; do + case $arg in + h) + usage + exit 1 + ;; + v) + VERSION=$OPTARG + ;; + s) + SNAPSHOT=$OPTARG + ;; + o) + OUTPUT=$OPTARG + ;; + a) + ARCHITECTURE=$OPTARG + ;; + :) + echo "Error: -${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${arg}" + exit 1 + ;; + esac +done + +if [ -z "$VERSION" ]; then + echo "Error: You must specify the OpenSearch version" + usage + exit 1 +fi + +[[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT +[ -z "$OUTPUT" ] && OUTPUT=artifacts + +mkdir -p $OUTPUT/plugins + +./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -x ktlint + +zipPath=$(find . -path \*build/distributions/*.zip) +distributions="$(dirname "${zipPath}")" + +echo "COPY ${distributions}/*.zip" +cp ${distributions}/*.zip ./$OUTPUT/plugins + +./gradlew publishShadowPublicationToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -x ktlint + +mkdir -p $OUTPUT/maven/org/opensearch +cp -r ./notification/build/libs $OUTPUT/maven/org/opensearch/notification +