From 2bc22d8b6adf60171b418ad67ddf19bf5a781100 Mon Sep 17 00:00:00 2001 From: QU3B1M Date: Mon, 30 Dec 2024 14:21:01 -0300 Subject: [PATCH] Add bash tool to copy the generated ECS templates to the plugins repo Update GHA to use the new tool Signed-off-by: --unset <--unset> Update ecs README with new tool usage Signed-off-by: --unset <--unset> Update ECS generator script name Signed-off-by: --unset <--unset> Remove unused index upload option Fix issue with exit codes on branch checking Signed-off-by: --unset <--unset> --- .github/workflows/generate-ecs-mappings.yml | 140 +------------ ecs/README.md | 93 +-------- ecs/generator/images/Dockerfile | 10 +- ecs/generator/images/generator.sh | 8 +- ecs/scripts/generate-pr-to-plugins.sh | 209 ++++++++++++++++++++ 5 files changed, 234 insertions(+), 226 deletions(-) create mode 100644 ecs/scripts/generate-pr-to-plugins.sh diff --git a/.github/workflows/generate-ecs-mappings.yml b/.github/workflows/generate-ecs-mappings.yml index 0da72230091c9..4b4d5bd63989f 100644 --- a/.github/workflows/generate-ecs-mappings.yml +++ b/.github/workflows/generate-ecs-mappings.yml @@ -16,140 +16,20 @@ jobs: with: fetch-depth: 2 + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + id: branch-name + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Set up Docker Compose run: sudo apt-get install docker-compose - - name: Extract ECS Modules and Run ECS Generator - id: run-ecs-generator - run: | - # Fetch base branch - git fetch origin +refs/heads/master:refs/remotes/origin/master - - # Extract the ECS module names from the modified files - modified_files=$(git diff --name-only origin/master) - updated_modules=() - for file in $modified_files; do - if [[ $file == ecs/* ]]; then - ecs_module=$(echo $file | cut -d'/' -f2) - if [[ ! " ${updated_modules[*]} " =~ " ${ecs_module} " ]]; then - updated_modules+=("$ecs_module") - fi - fi - done - - # Filter out modules that do not have corresponding JSON files - declare -A module_to_file=( - [agent]="index-template-agent.json" - [alerts]="index-template-alerts.json" - [commands]="index-template-commands.json" - [hardware]="index-template-hardware.json" - [hotfixes]="index-template-hotfixes.json" - [fim]="index-template-fim.json" - [networks]="index-template-networks.json" - [packages]="index-template-packages.json" - [ports]="index-template-ports.json" - [processes]="index-template-processes.json" - [scheduled-commands]="index-template-scheduled-commands.json" - [system]="index-template-system.json" - [vulnerabilities]="index-template-vulnerabilities.json" - ) - - relevant_modules=() - for ecs_module in "${updated_modules[@]}"; do - if [[ -n "${module_to_file[$ecs_module]}" ]]; then - relevant_modules+=("$ecs_module") - fi - done - - if [[ ${#relevant_modules[@]} -gt 0 ]]; then - export REPO_PATH=$(pwd) - for ecs_module in "${relevant_modules[@]}"; do - # Run the ECS generator script for each relevant module - bash ecs/generator/mapping-generator.sh run "$ecs_module" - echo "Processed ECS module: $ecs_module" - done - echo "relevant_modules=${relevant_modules[*]}" >> $GITHUB_ENV - else - echo "No relevant modifications detected in ecs/ directory." - exit 0 - fi - - - name: Tear down ECS Generator - if: always() - run: bash ecs/generator/mapping-generator.sh down - - - name: Upload artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: ecs-template - path: ecs/**/mappings/v8.11.0/generated/elasticsearch/legacy/template.json - - - name: Checkout target repository - uses: actions/checkout@v4 - with: - repository: wazuh/wazuh-indexer-plugins - token: ${{ secrets.GITHUB_TOKEN }} - path: wazuh-indexer-plugins - - - name: Copy generated files to target repository - run: | - # Map ECS modules to target JSON filenames - declare -A module_to_file=( - [agent]="index-template-agent.json" - [alerts]="index-template-alerts.json" - [commands]="index-template-commands.json" - [hardware]="index-template-hardware.json" - [hotfixes]="index-template-hotfixes.json" - [fim]="index-template-fim.json" - [networks]="index-template-networks.json" - [packages]="index-template-packages.json" - [ports]="index-template-ports.json" - [processes]="index-template-processes.json" - [scheduled-commands]="index-template-scheduled-commands.json" - [system]="index-template-system.json" - [vulnerabilities]="index-template-vulnerabilities.json" - ) - - for ecs_module in ${relevant_modules[@]}; do - target_file=${module_to_file[$ecs_module]} - if [[ -z "$target_file" ]]; then - echo "No corresponding file for module $ecs_module" - continue - fi - - mkdir -p wazuh-indexer-plugins/plugins/setup/src/main/resources/ - cp ecs/$ecs_module/mappings/v8.11.0/generated/elasticsearch/legacy/template.json wazuh-indexer-plugins/plugins/setup/src/main/resources/$target_file - done - - - name: Commit and push changes + - name: Generate PR to wazuh-indxer-plugins + env: + GITHUB_TOKEN: ${{ secrets.ACTION_TOKEN }} run: | - cd wazuh-indexer-plugins - git config --global user.email "github-actions@github.com" - git config --global user.name "GitHub Actions" - - branch_name="update-ecs-templates" - - # Check if branch exists - if git ls-remote --heads origin $branch_name | grep $branch_name; then - git checkout $branch_name - else - git checkout -b $branch_name - fi - - git add . - git commit -m "Update ECS templates for modified modules: $relevant_modules" - git push origin $branch_name - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "Update ECS templates for modified modules: $relevant_modules" - branch: update-ecs-templates - title: "Update ECS templates for modified modules: $relevant_modules" - body: "This PR updates the ECS templates for the following modules: $relevant_modules." - base: master + bash ecs/scripts/generate-pr-to-plugins.sh \ + -b ${{ steps.branch-name.outputs.branch }} diff --git a/ecs/README.md b/ecs/README.md index 35e4e783bbd98..074e04a574a80 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -4,9 +4,8 @@ This script generates the ECS mappings for the Wazuh indices. ### Requirements -- ECS repository clone. The script is meant to be launched from the root level of that repository. -- `Python` 3.6 or higher + `venv` module -- `jq` +- [Docker Desktop](https://docs.docker.com/desktop/setup/install/linux/) + > Other option is to install the [docker-compose plugin](https://docs.docker.com/compose/install/#scenario-two-install-the-docker-compose-plugin). ### Folder structure @@ -15,67 +14,21 @@ files to generate the mappings. These are the inputs for the ECS generator. ### Usage -1. Get a copy of the ECS repository at the same level as the `wazuh-indexer` repo: - - ```console - git clone git@github.com:elastic/ecs.git - ``` - -2. Install the dependencies: - - ```console - cd ecs - python3 -m venv env - source env/bin/activate - pip install -r scripts/requirements.txt - ``` - -2. Copy the `generate.sh` script to the root level of the ECS repository. - - ```console - cp generate.sh ../../ecs - cd ../../ecs - bash generate.sh - ``` - - Expected output: - ``` - Usage: generate.sh [--upload ] - * ECS_VERSION: ECS version to generate mappings for - * INDEXER_SRC: Path to the wazuh-indexer repository - * MODULE: Module to generate mappings for - * --upload : Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200 - Example: generate.sh v8.11.0 ~/wazuh-indexer states-vulnerabilities --upload https://indexer:9200 - ``` - -3. Use the `generate.sh` script to generate the mappings for a module. The script takes 3 arguments, -plus 2 optional arguments to upload the mappings to the `wazuh-indexer`. Both, composable and legacy mappings -are generated. For example, to generate the mappings for the `states-vulnerabilities` module using the - ECS version `v8.11.0` and assuming that path of this repository is `~/wazuh/wazuh-indexer`: - +1. Execute the mapping-generator tool ```bash - ./generate.sh v8.11.0 ~/wazuh/wazuh-indexer states-vulnerabilities - ``` - - The tool will output the folder where they have been generated. - - ```console - Loading schemas from git ref v8.11.0 - Running generator. ECS version 8.11.0 - Mappings saved to ~/wazuh/wazuh-indexer/ecs/states-vulnerabilities/mappings/v8.11.0 + bash ecs/generator/mapping-generator.sh run ``` - -4. When you are done. Exit the virtual environment. - - ```console - deactivate +2. (Optional) Run the tool's cleanup + > The tool stops the container automatically, but it is recommended to run the down command if the tool is not going to be used anymore. + ```bash + bash ecs/generator/mapping-generator.sh down ``` ### Output A new `mappings` folder will be created inside the module folder, containing all the generated files. The files are versioned using the ECS version, so different versions of the same module can be generated. -For our use case, the most important files are under `mappings//generated/elasticsearch/legacy/`: +For our use case, the most important files are under `mappings/v8.11.0/generated/elasticsearch/legacy/`: - `template.json`: Elasticsearch compatible index template for the module - `opensearch-template.json`: OpenSearch compatible index template for the module @@ -137,31 +90,3 @@ The script uses log file. Check it out for debugging or additional information. - [ECS repository](https://github.com/elastic/ecs) - [ECS usage](https://github.com/elastic/ecs/blob/main/USAGE.md) - [ECS field reference](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) - -### All-in-one script - -```bash -#!/bin/bash - -indices=( - agent - alerts - command - states-fim - states-inventory-hardware - states-inventory-hotfixes - states-inventory-networks - states-inventory-packages - states-inventory-ports - states-inventory-processes - states-inventory-system - states-vulnerabilities -) - -ECS="v8.11.0" -WI_REPO_PATH=~/wazuh/wazuh-indexer - -for index in "${indices[@]}"; do - bash generate.sh $ECS $WI_REPO_PATH "$index" -done -``` diff --git a/ecs/generator/images/Dockerfile b/ecs/generator/images/Dockerfile index 0153810699146..0b0c882ef42e8 100644 --- a/ecs/generator/images/Dockerfile +++ b/ecs/generator/images/Dockerfile @@ -9,22 +9,22 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ # Clone elastic ECS repository and install required Python libraries - git clone https://github.com/elastic/ecs.git && \ + git clone https://github.com/elastic/ecs.git -b v8.11.0 --depth 1 && \ pip install -r ecs/scripts/requirements.txt && \ # Create the directory for the ecs definitions (this will be used as a volume) mkdir -p /source/ecs -# Ensure the generate.sh script is in the correct location -ADD docker/ecs/images/generator.sh /ecs/generator.sh +# Ensure the generator.sh script is in the correct location +ADD ecs/generator/images/generator.sh /ecs/generator.sh # Define the directory as a volume to allow for external mounting VOLUME /source/ecs -# Ensure the generate.sh script is executable +# Ensure the generator.sh script is executable RUN chmod +x /ecs/generator.sh # Set the working directory to the ECS repository WORKDIR /ecs -# Define the entry point for the container to execute the generate.sh script +# Define the entry point for the container to execute the generator.sh script ENTRYPOINT ["/bin/bash", "/ecs/generator.sh"] diff --git a/ecs/generator/images/generator.sh b/ecs/generator/images/generator.sh index 3958150e2db9f..2b6b3c05f0068 100755 --- a/ecs/generator/images/generator.sh +++ b/ecs/generator/images/generator.sh @@ -10,8 +10,6 @@ set -euo pipefail # Default values ECS_VERSION="${ECS_VERSION:-v8.11.0}" ECS_SOURCE="${ECS_SOURCE:-/source}" -UPLOAD="${UPLOAD:-false}" -URL="${URL:-https://localhost:9200}" # Function to display usage information show_usage() { @@ -20,8 +18,6 @@ show_usage() { echo " * ECS_MODULE: Module to generate mappings for" echo " * ECS_VERSION: (Optional) ECS version to generate mappings for (default: v8.11.0)" echo " * ECS_SOURCE: (Optional) Path to the wazuh-indexer repository (default: /source)" - echo " * UPLOAD: (Optional) Upload generated index template to the Wazuh Indexer cluster (default: false)" - echo " * URL: (Optional) URL of the Wazuh Indexer cluster (default: https://localhost:9200)" echo "Example: docker run -e ECS_MODULE=alerts -e ECS_VERSION=v8.11.0 ecs-generator" } @@ -52,8 +48,6 @@ generate_mappings() { local ecs_module="$1" local indexer_path="$2" local ecs_version="$3" - local upload="$4" - local url="$5" local in_files_dir="$indexer_path/ecs/$ecs_module/fields" local out_dir="$indexer_path/ecs/$ecs_module/mappings/$ecs_version" @@ -106,4 +100,4 @@ generate_mappings() { } # Generate mappings -generate_mappings "$ECS_MODULE" "$ECS_SOURCE" "$ECS_VERSION" "$UPLOAD" "$URL" +generate_mappings "$ECS_MODULE" "$ECS_SOURCE" "$ECS_VERSION" diff --git a/ecs/scripts/generate-pr-to-plugins.sh b/ecs/scripts/generate-pr-to-plugins.sh new file mode 100644 index 0000000000000..e92cb0da2422c --- /dev/null +++ b/ecs/scripts/generate-pr-to-plugins.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +# Constants +MAPPINGS_SUBPATH="mappings/v8.11.0/generated/elasticsearch/legacy/template.json" +TEMPLATES_PATH="plugins/setup/src/main/resources/" +PLUGINS_REPO="wazuh/wazuh-indexer-plugins" +CURRENT_PATH=$(pwd) +BASE_BRANCH=${BASE_BRANCH:-master} +PLUGINS_LOCAL_PATH=${PLUGINS_LOCAL_PATH:-"$CURRENT_PATH"/../wazuh-indexer-plugins} +# Global variables +declare -a relevant_modules +declare -A module_to_file + +command_exists() { + command -v "$1" &> /dev/null +} + +validate_dependencies() { + local required_commands=("docker" "docker-compose" "gh") + for cmd in "${required_commands[@]}"; do + if ! command_exists "$cmd"; then + echo "Error: $cmd is not installed. Please install it and try again." + exit 1 + fi + done +} + +fetch_and_extract_modules() { + echo + echo "---> Fetching and extracting modified ECS modules..." + git fetch origin +refs/heads/master:refs/remotes/origin/master + local modified_files + local updated_modules=() + modified_files=$(git diff --name-only origin/"$BASE_BRANCH") + + for file in $modified_files; do + if [[ $file == ecs/* ]]; then + ecs_module=$(echo "$file" | cut -d'/' -f2) + if [[ ! " ${updated_modules[*]} " =~ ${ecs_module} ]]; then + updated_modules+=("$ecs_module") + fi + fi + done + echo "Updated ECS modules: ${updated_modules[*]}" + + # Mapping section + module_to_file=( + [agent]="index-template-agent.json" + [alerts]="index-template-alerts.json" + [commands]="index-template-commands.json" + [states-fim]="index-template-fim.json" + [states-inventory-hardware]="index-template-hardware.json" + [states-inventory-hotfixes]="index-template-hotfixes.json" + [states-inventory-networks]="index-template-networks.json" + [states-inventory-packages]="index-template-packages.json" + [states-inventory-ports]="index-template-ports.json" + [states-inventory-processes]="index-template-processes.json" + [states-inventory-scheduled-commands]="index-template-scheduled-commands.json" + [states-inventory-system]="index-template-system.json" + [states-vulnerabilities]="index-template-vulnerabilities.json" + ) + + relevant_modules=() + for ecs_module in "${updated_modules[@]}"; do + if [[ -n "${module_to_file[$ecs_module]}" ]]; then + relevant_modules+=("$ecs_module") + fi + done + echo "Relevant ECS modules: ${relevant_modules[*]}" +} + +run_ecs_generator() { + echo + echo "---> Running ECS Generator script..." + if [[ ${#relevant_modules[@]} -gt 0 ]]; then + for ecs_module in "${relevant_modules[@]}"; do + bash ecs/generator/mapping-generator.sh run "$ecs_module" + echo "Processed ECS module: $ecs_module" + bash ecs/generator/mapping-generator.sh down + done + else + echo "No relevant modifications detected in ecs/ directory." + exit 0 + fi +} + + +clone_target_repo() { + echo + echo "---> Cloning ${PLUGINS_REPO} repository..." + if [ ! -d "$PLUGINS_LOCAL_PATH" ]; then + git clone https://github.com/$PLUGINS_REPO.git "$PLUGINS_LOCAL_PATH" + fi + cd "$PLUGINS_LOCAL_PATH" || exit + git config --global user.email "github-actions@github.com" + git config --global user.name "GitHub Actions" + git pull + # Set up authentication with GitHub token + git remote set-url origin https://"$GITHUB_TOKEN"@github.com/$PLUGINS_REPO.git + # Set up the default remote URL + gh repo set-default https://"$GITHUB_TOKEN"@github.com/$PLUGINS_REPO.git +} + +commit_and_push_changes() { + echo + echo "---> Committing and pushing changes to ${PLUGINS_REPO} repository..." + git ls-remote --exit-code --heads origin "$branch_name" >/dev/null 2>&1 + EXIT_CODE=$? + + if [[ $EXIT_CODE == '0' ]]; then + git checkout "$branch_name" + git pull origin "$branch_name" + else + git checkout -b "$branch_name" + git push --set-upstream origin "$branch_name" + fi + + echo "Copying ECS templates to the plugins repository..." + for ecs_module in "${relevant_modules[@]}"; do + target_file=${module_to_file[$ecs_module]} + if [[ -z "$target_file" ]]; then + continue + fi + + mkdir -p $TEMPLATES_PATH + echo " - Copy template for module '$ecs_module' to '$target_file'" + cp "$CURRENT_PATH/ecs/$ecs_module/$MAPPINGS_SUBPATH" "$TEMPLATES_PATH/$target_file" + done + + git status --short + + if ! git diff-index --quiet HEAD --; then + echo "Changes detected. Committing and pushing to the repository..." + git add . + git commit -m "Update ECS templates for modified modules: ${relevant_modules[*]}" + git push + else + echo "Nothing to commit, working tree clean." + exit 0 + fi +} + +create_or_update_pr() { + echo + echo "---> Creating or updating Pull Request..." + + local existing_pr + local modules + local title + local body + + existing_pr=$(gh pr list --head "$branch_name" --json number --jq '.[].number') + modules=$(IFS=,; echo "${relevant_modules[*]}") + title="Update ECS templates for modified modules: ${modules}" + body="This PR updates the ECS templates for the following modules: ${modules}." + + if [ -z "$existing_pr" ]; then + gh pr create \ + --title "$title" \ + --body "$body" \ + --base master \ + --head "$branch_name" + else + echo "PR already exists: $existing_pr. Updating the PR..." + gh pr edit "$existing_pr" \ + --title "$title" \ + --body "$body" + fi +} + +usage() { + echo "Usage: $0 -b -t " + echo " -b Branch name to create or update the PR." + echo " -t [GITHUB_TOKEN] (Optional) GitHub token to authenticate with GitHub API." + echo " If not provided, the script will use the GITHUB_TOKEN environment variable." + exit 1 +} + +main() { + while getopts ":b:t:" opt; do + case ${opt} in + b ) + branch_name=$OPTARG + ;; + t ) + GITHUB_TOKEN=$OPTARG + ;; + \? ) + usage + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + usage + ;; + esac + done + if [ -z "$branch_name" ] || [ -z "$GITHUB_TOKEN" ]; then + usage + fi + + validate_dependencies + fetch_and_extract_modules + run_ecs_generator # Exit if no changes on relevant modules. + clone_target_repo + commit_and_push_changes # Exit if no changes detected. + create_or_update_pr +} + +main "$@"