feat: add option to use s3 compatible storage for snapshot management

Allow to build `wazuh-indexer` docker image with additional plugin `repository-s3` installed, via a new optional `--s3-repository` parameter. Parameter `--s3-repository` is set to `false` by default. On `wazuh-indexer` container startup, if `repository-s3` plugin is installed, the system will attempt to read `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables and add their values to the OpenSearch keystore.

For an S3 compatible storage to be configured as a snapshot repository correctly, some additional steps must be taken, mainly:
1. `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_REGION` environment variables must be added to `docker-compose.yml` for the `wazuh.indexer` service. E.g.:
```yaml
...
  wazuh.indexer:
    ...
    environment:
      ...
      - "AWS_ACCESS_KEY_ID=exampleAccessKeyID"
      - "AWS_SECRET_ACCESS_KEY=exampleSecretAccessKey"
      - "AWS_REGION=us-west-1"
...
```
2. Additional settings for the S3 compatible storage must be added to `config/wazuh_indexer/wazuh.indexer.yml`. E.g.:
```yaml
...
s3.client.default.endpoint: minio:9000
s3.client.default.protocol: http
s3.client.default.max_retries: 3
s3.client.default.read_timeout: 50s
s3.client.default.path_style_access: true
s3.client.default.use_throttle_retries: true
```

See https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore/ for more information.

README.md

@@ -47,6 +47,10 @@ FILEBEAT_SSL_VERIFICATION_MODE=full                 # Filebeat SSL Verification
 SSL_CERTIFICATE_AUTHORITIES=""                      # Path of Filebeat SSL CA
 SSL_CERTIFICATE=""                                  # Path of Filebeat SSL Certificate
 SSL_KEY=""                                          # Path of Filebeat SSL Key
+
+AWS_ACCESS_KEY_ID=""                                # AWS access key for an S3 Compatible Storage
+AWS_SECRET_ACCESS_KEY=""                            # AWS secret key for an S3 Compatible Storage
+AWS_REGION=""                                       # AWS region for an S3 Compatible Storage
 ```
 
 ### Dashboard

build-docker-images/README.md

@@ -26,6 +26,7 @@ Usage: build-docker-images/build-images.sh [OPTIONS]
     -d, --dev <ref>              [Optional] Set the development stage you want to build, example rc1 or beta1, not used by default.
     -f, --filebeat-module <ref>  [Optional] Set Filebeat module version. By default 0.4.
     -r, --revision <rev>         [Optional] Package revision. By default 1
+    -s, --s3-repository          [Optional] Install 'repository-s3' plugin for OpenSearch. By default false.
     -v, --version <ver>          [Optional] Set the Wazuh version should be builded. By default, 5.0.0.
     -h, --help                   Show this help.
 

build-docker-images/build-images.sh

@@ -16,6 +16,7 @@ WAZUH_IMAGE_VERSION="5.0.0"
 WAZUH_TAG_REVISION="1"
 WAZUH_DEV_STAGE=""
 FILEBEAT_MODULE_VERSION="0.4"
+S3_REPOSITORY="false"
 
 # -----------------------------------------------------------------------------
 
@@ -69,6 +70,7 @@ build() {
     echo FILEBEAT_TEMPLATE_BRANCH=$FILEBEAT_TEMPLATE_BRANCH >> .env
     echo WAZUH_FILEBEAT_MODULE=$WAZUH_FILEBEAT_MODULE >> .env
     echo WAZUH_UI_REVISION=$WAZUH_UI_REVISION >> .env
+    echo S3_REPOSITORY=$S3_REPOSITORY >> .env
 
     docker-compose -f build-docker-images/build-images.yml --env-file .env build --no-cache
     docker build -t wazuh/wazuh-cert-tool:$WAZUH_IMAGE_VERSION build-docker-images/cert-tool-image/
@@ -85,6 +87,7 @@ help() {
     echo "    -d, --dev <ref>              [Optional] Set the development stage you want to build, example rc1 or beta1, not used by default."
     echo "    -f, --filebeat-module <ref>  [Optional] Set Filebeat module version. By default ${FILEBEAT_MODULE_VERSION}."
     echo "    -r, --revision <rev>         [Optional] Package revision. By default ${WAZUH_TAG_REVISION}"
+    echo "    -s, --s3-repository          [Optional] Install 'repository-s3' plugin for OpenSearch. By default ${S3_REPOSITORY}."
     echo "    -v, --version <ver>          [Optional] Set the Wazuh version should be builded. By default, ${WAZUH_IMAGE_VERSION}."
     echo "    -h, --help                   Show this help."
     echo
@@ -116,6 +119,10 @@ main() {
                 help 1
             fi
             ;;
+        "-s"|"--s3-repository")
+            S3_REPOSITORY="true"
+            shift 1
+            ;;
         "-r"|"--revision")
             if [ -n "${2}" ]; then
                 WAZUH_TAG_REVISION="${2}"

build-docker-images/build-images.yml

@@ -42,6 +42,7 @@ services:
       args:
         WAZUH_VERSION: ${WAZUH_VERSION}
         WAZUH_TAG_REVISION: ${WAZUH_TAG_REVISION}
+        S3_REPOSITORY: ${S3_REPOSITORY}
     image: wazuh/wazuh-indexer:${WAZUH_IMAGE_VERSION}
     hostname: wazuh.indexer
     restart: always

build-docker-images/wazuh-indexer/Dockerfile

@@ -30,10 +30,13 @@ RUN bash config.sh
 ################################################################################
 FROM amazonlinux:2023
 
+ARG S3_REPOSITORY
+
 ENV USER="wazuh-indexer" \
     GROUP="wazuh-indexer" \
     NAME="wazuh-indexer" \
-    INSTALL_DIR="/usr/share/wazuh-indexer"
+    INSTALL_DIR="/usr/share/wazuh-indexer" \
+    OPENSEARCH_PATH_CONF="/usr/share/wazuh-indexer"
 
 # Set $JAVA_HOME
 RUN echo "export JAVA_HOME=$INSTALL_DIR/jdk" >> /etc/profile.d/java_home.sh && \
@@ -85,6 +88,9 @@ RUN mkdir -p /var/lib/wazuh-indexer && chown 1000:1000 /var/lib/wazuh-indexer &&
 
 USER wazuh-indexer
 
+# Allow to use S3 Compatible Storage as a snapshot repository
+RUN if [ "$S3_REPOSITORY" = "true" ] ; then "${INSTALL_DIR}/bin/opensearch-plugin" install --batch repository-s3 ; fi
+
 # Services ports
 EXPOSE 9200
 

build-docker-images/wazuh-indexer/config/entrypoint.sh

@@ -367,6 +367,36 @@ if [[ -f bin/opensearch-users ]]; then
   fi
 fi
 
+# Allow to use S3 Compatible Storage as a snapshot repository
+#
+# Check if AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables
+# are set and OpenSearch plugin 'repository-s3' is installed.
+if [[ -n "$AWS_ACCESS_KEY_ID" && -n "$AWS_SECRET_ACCESS_KEY" ]] && \
+    (run_as_other_user_if_needed "${INSTALLATION_DIR}/bin/opensearch-plugin" list | grep -q '^repository-s3$') ; then
+  [[ -f /usr/share/wazuh-indexer/opensearch.keystore ]] || (run_as_other_user_if_needed "${INSTALLATION_DIR}/bin/opensearch-keystore" create)
+  if ! (run_as_other_user_if_needed "${INSTALLATION_DIR}/bin/opensearch-keystore" has-passwd --silent) ; then
+    # keystore is unencrypted
+    if ! (run_as_other_user_if_needed "${INSTALLATION_DIR}/bin/opensearch-keystore" list | grep -q '^s3.client.default.access_key$') ; then
+      (run_as_other_user_if_needed echo "$AWS_ACCESS_KEY_ID" | "${INSTALLATION_DIR}/bin/opensearch-keystore" add -x 's3.client.default.access_key')
+    fi
+    if ! (run_as_other_user_if_needed "${INSTALLATION_DIR}/bin/opensearch-keystore" list | grep -q '^s3.client.default.secret_key$') ; then
+      (run_as_other_user_if_needed echo "$AWS_SECRET_ACCESS_KEY" | "${INSTALLATION_DIR}/bin/opensearch-keystore" add -x 's3.client.default.secret_key')
+    fi
+  else
+    # keystore requires password
+    if ! (run_as_other_user_if_needed echo "$KEYSTORE_PASSWORD" \
+        | "${INSTALLATION_DIR}/bin/opensearch-keystore" list | grep -q '^s3.client.default.access_key$') ; then
+      COMMANDS="$(printf "%s\n%s" "$KEYSTORE_PASSWORD" "$AWS_ACCESS_KEY_ID")"
+      (run_as_other_user_if_needed echo "$COMMANDS" | "${INSTALLATION_DIR}/bin/opensearch-keystore" add -x 's3.client.default.access_key')
+    fi
+    if ! (run_as_other_user_if_needed echo "$KEYSTORE_PASSWORD" \
+        | "${INSTALLATION_DIR}/bin/opensearch-keystore" list | grep -q '^s3.client.default.secret_key$') ; then
+      COMMANDS="$(printf "%s\n%s" "$KEYSTORE_PASSWORD" "$AWS_SECRET_ACCESS_KEY")"
+      (run_as_other_user_if_needed echo "$COMMANDS" | "${INSTALLATION_DIR}/bin/opensearch-keystore" add -x 's3.client.default.secret_key')
+    fi
+  fi
+fi
+
 if [[ "$(id -u)" == "0" ]]; then
   # If requested and running as root, mutate the ownership of bind-mounts
   if [[ -n "$TAKE_FILE_OWNERSHIP" ]]; then