Merge pull request #113 from github/hoenn/run-es8-ci

Add ES8 tests, add test data for breaking change

integration_data/v6/documents.dat

@@ -0,0 +1,20 @@
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }

integration_data/v7/documents.dat

@@ -0,0 +1,20 @@
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test", "_type" : "testdoc" } }
+{ "stat" : "test_data", "value": "foo" }

integration_data/v8/documents.dat

@@ -0,0 +1,20 @@
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test"} }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }
+{ "index" : { "_index" : "integration_test" } }
+{ "stat" : "test_data", "value": "foo" }

integration_data/v8/elasticsearch-cluster-compose.yml

@@ -0,0 +1,82 @@
+version: '3'
+
+services:
+  elasticsearch-v8:
+    build:
+      context: .
+      dockerfile: elasticsearch.dockerfile
+    image: vulcanizer-elasticsearch:v8
+    container_name: vulcanizer-elasticsearch-v8
+    environment:
+      - cluster.name=vulcanizer-elasticsearch-v8
+      - node.name=vulcanizer-elasticsearch-v8
+      - bootstrap.memory_lock=true
+      - "path.repo=/backups"
+      - "discovery.seed_hosts=vulcanizer-elasticsearch-v8,vulcanizer-elasticsearch-v8-2"
+      - "cluster.initial_master_nodes=vulcanizer-elasticsearch-v8,vulcanizer-elasticsearch-v8-2"
+      - "ES_JAVA_OPTS=-Xms512M -Xmx512M"
+      - xpack.security.enabled=false
+      - xpack.profiling.enabled=false
+      - xpack.ml.enabled=false
+      - xpack.graph.enabled=false
+      - xpack.watcher.enabled=false
+      - ingest.geoip.downloader.enabled=false
+      - indices.lifecycle.history_index_enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    cap_add:
+      - IPC_LOCK
+    ports:
+      - 49200:9200
+    networks:
+      - vulcanizer-esnet
+    volumes:
+      - vulcanizer-backup-volume:/backups
+
+  elasticsearch-v8-2:
+    build:
+      context: .
+      dockerfile: elasticsearch.dockerfile
+    image: vulcanizer-elasticsearch:v8
+    depends_on:
+      - "elasticsearch-v8"
+    container_name: vulcanizer-elasticsearch-v8-2
+    environment:
+      - cluster.name=vulcanizer-elasticsearch-v8
+      - bootstrap.memory_lock=true
+      - node.name=vulcanizer-elasticsearch-v8-2
+      - "path.repo=/backups"
+      - "discovery.seed_hosts=vulcanizer-elasticsearch-v8,vulcanizer-elasticsearch-v8-2"
+      - "cluster.initial_master_nodes=vulcanizer-elasticsearch-v8,vulcanizer-elasticsearch-v8-2"
+      - "ES_JAVA_OPTS=-Xms512M -Xmx512M"
+      - xpack.security.enabled=false
+      - xpack.profiling.enabled=false
+      - xpack.ml.enabled=false
+      - xpack.graph.enabled=false
+      - xpack.watcher.enabled=false
+      - ingest.geoip.downloader.enabled=false
+      - indices.lifecycle.history_index_enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    cap_add:
+      - IPC_LOCK
+    networks:
+      - vulcanizer-esnet
+    volumes:
+      - vulcanizer-backup-volume:/backups
+
+networks:
+  vulcanizer-esnet:
+
+volumes:
+  vulcanizer-backup-volume:

integration_data/v8/elasticsearch.dockerfile

@@ -0,0 +1,7 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch:8.13.0
+
+USER root
+
+RUN mkdir /backups && chown elasticsearch:elasticsearch /backups
+
+USER elasticsearch

integration_test.go

@@ -1,11 +1,13 @@
+//go:build integration
 // +build integration
 
 package vulcanizer_test
 
 import (
-	"github.com/github/vulcanizer"
 	"testing"
 	"time"
+
+	"github.com/github/vulcanizer"
 )
 
 func TestNodes(t *testing.T) {

script/integration-test

@@ -17,20 +17,21 @@ fi
 # Run regular unit tests first
 ./script/test
 
-elasticsearch_versions=(v5 v6 v7)
+# These match integration_data/ sub-folder names
+elasticsearch_versions=(v5 v6 v7 v8)
 
 for v in "${elasticsearch_versions[@]}"
 do
   # Make sure everything is clear from previous runs
-  docker-compose -f integration_data/$v/elasticsearch-cluster-compose.yml down
+  docker compose -f integration_data/$v/elasticsearch-cluster-compose.yml down
   docker volume rm ${v}_vulcanizer-backup-volume || echo "OK for volume not to exist"
 done
 
 for v in "${elasticsearch_versions[@]}"
 do
   cd integration_data/$v
   echo "Running integration tests for Elasticsearch $v"
-  docker-compose -f elasticsearch-cluster-compose.yml up --build -d
+  docker compose -f elasticsearch-cluster-compose.yml up --build -d
   echo "Wait for Elasticsearch $v to start..."
   until foo=$(curl -s localhost:49200)
   do
@@ -44,7 +45,7 @@ do
     }
   }'
   echo "Filling in data to integration_test index"
-  curl -s -H "Content-Type: application/x-ndjson" -XPOST localhost:49200/_bulk --data-binary "@../documents.dat"; echo
+  curl -s -H "Content-Type: application/x-ndjson" -XPOST localhost:49200/_bulk --data-binary "@documents.dat"; echo
   echo "Creating snapshot repository backup-repo"
   curl -H "Content-Type: application/json" -XPUT localhost:49200/_snapshot/backup-repo -d '{ "type": "fs", "settings": { "location": "/backups" } }'
   echo "Making snapshot snapshot_1 in repository backup-repo"
@@ -55,6 +56,6 @@ do
   # Run tests
   go test -v github.com/github/vulcanizer/... -tags integration -count=1
 
-  docker-compose -f elasticsearch-cluster-compose.yml down
+  docker compose -f elasticsearch-cluster-compose.yml down
   cd ../../
 done