Add USE_OLD_LOCALES test (#821)

The newly added test verifies that with the old locales (Ubuntu 18.04) kept postgres sorting order is actually preserved on Ubuntu 22.04. The test is only run on push to master branch.
zalando · Dec 28, 2022 · dffd18f · dffd18f
1 parent 8f15e86
commit dffd18f
Show file tree

Hide file tree

Showing 6 changed files with 234 additions and 46 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -28,3 +28,6 @@ jobs:
       run: cd postgres-appliance && docker build -t spilo .
     - name: Test spilo docker image
       run: bash postgres-appliance/tests/test_spilo.sh
+    - name: Test USE_OLD_LOCALES
+      run: bash -x postgres-appliance/tests/locales_test/test_locales.sh
+      if: github.event_name == 'push'
diff --git a/postgres-appliance/tests/locales_test/generate_data.sh b/postgres-appliance/tests/locales_test/generate_data.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -ex
+
+cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
+
+readonly container=$1
+readonly output_file=$2
+
+
+function generate_data() {
+    docker_exec "$container" $'cd $PGDATA;
+        rm -rf locales_test; mkdir locales_test; cd locales_test;
+        /bin/bash "/home/postgres/tests/helper_script.sh";
+        truncate -s -1 _base-characters \
+        && psql -c "insert into chars select regexp_split_to_table(pg_read_file(\'locales_test/_base-characters\')::text, E\'\\n\');"
+    '
+}
+
+# Create an auxiliary table
+docker_exec "$container" "psql -d postgres -c 'drop table if exists chars; create table chars(chr text);'"
+
+# Insert data into the auxiliary table
+generate_data
+
+# Write sorted data to an output file
+docker_exec "$container" "psql -c '\copy (select * from chars order by 1) to ${output_file}'"
diff --git a/postgres-appliance/tests/locales_test/helper_script.sh b/postgres-appliance/tests/locales_test/helper_script.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+
+# Script is copied from https://github.com/ardentperf/glibc-unicode-sorting
+
+UNICODE_VERS="14"
+curl -kO https://www.unicode.org/Public/${UNICODE_VERS}.0.0/ucd/UnicodeData.txt
+
+perl -naF';' -CO -e'
+  use utf8;
+  sub pr {
+    print chr($_[0]) . "\n";  # 199
+    print chr($_[0]) . "B\n";              # 200
+    print chr($_[0]) . "O\n";              # 201
+    print chr($_[0]) . "3\n";              # 202
+    print chr($_[0]) . ".\n";              # 203
+    print chr($_[0]) . " \n";              # 204
+    print chr($_[0]) . "様\n";              # 205
+    print chr($_[0]) . "ク\n";              # 206
+    print "B" . chr($_[0]) . "\n";         # 210
+    print "O" . chr($_[0]) . "\n";         # 211
+    print "3" . chr($_[0]) . "\n";         # 212
+    print "." . chr($_[0]) . "\n";         # 213
+    print " " . chr($_[0]) . "\n";         # 214
+    print "様" . chr($_[0]) . "\n";         # 215
+    print "ク" . chr($_[0]) . "\n";         # 216
+    print chr($_[0]) . chr($_[0]) . "\n";  # 299
+    print chr($_[0]) . "BB\n";                          # 300
+    print chr($_[0]) . "OO\n";                          # 301
+    print chr($_[0]) . "33\n";                          # 302
+    print chr($_[0]) . "..\n";                          # 303
+    print chr($_[0]) . "  \n";                          # 304
+    print chr($_[0]) . "様様\n";                          # 305
+    print chr($_[0]) . "クク\n";                          # 306
+    print "B" . chr($_[0]) . "B\n";                     # 310
+    print "O" . chr($_[0]) . "O\n";                     # 311
+    print "3" . chr($_[0]) . "3\n";                     # 312
+    print "." . chr($_[0]) . ".\n";                     # 313
+    print " " . chr($_[0]) . " \n";                     # 314
+    print "様" . chr($_[0]) . "様\n";                     # 315
+    print "ク" . chr($_[0]) . "ク\n";                     # 316
+    print "BB" . chr($_[0]) . "\n";                     # 320
+    print "OO" . chr($_[0]) . "\n";                     # 321
+    print "33" . chr($_[0]) . "\n";                     # 322
+    print ".." . chr($_[0]) . "\n";                     # 323
+    print "  " . chr($_[0]) . "\n";                     # 324
+    print "様様" . chr($_[0]) . "\n";                     # 325
+    print "クク" . chr($_[0]) . "\n";                     # 326
+    print chr($_[0]) . chr($_[0]) . "B\n";              # 330
+    print chr($_[0]) . chr($_[0]) . "O\n";              # 331
+    print chr($_[0]) . chr($_[0]) . "3\n";              # 332
+    print chr($_[0]) . chr($_[0]) . ".\n";              # 333
+    print chr($_[0]) . chr($_[0]) . " \n";              # 334
+    print chr($_[0]) . chr($_[0]) . "様\n";              # 335
+    print chr($_[0]) . chr($_[0]) . "ク\n";              # 336
+    print chr($_[0]) . "B" . chr($_[0]) . "\n";         # 340
+    print chr($_[0]) . "O" . chr($_[0]) . "\n";         # 341
+    print chr($_[0]) . "3" . chr($_[0]) . "\n";         # 342
+    print chr($_[0]) . "." . chr($_[0]) . "\n";         # 343
+    print chr($_[0]) . " " . chr($_[0]) . "\n";         # 344
+    print chr($_[0]) . "様" . chr($_[0]) . "\n";         # 345
+    print chr($_[0]) . "ク" . chr($_[0]) . "\n";         # 346
+    print "B" . chr($_[0]) . chr($_[0]) . "\n";         # 350
+    print "O" . chr($_[0]) . chr($_[0]) . "\n";         # 351
+    print "3" . chr($_[0]) . chr($_[0]) . "\n";         # 352
+    print "." . chr($_[0]) . chr($_[0]) . "\n";         # 353
+    print " " . chr($_[0]) . chr($_[0]) . "\n";         # 354
+    print "様" . chr($_[0]) . chr($_[0]) . "\n";         # 355
+    print "ク" . chr($_[0]) . chr($_[0]) . "\n";         # 356
+    print "3B" . chr($_[0]) . "\n";                     # 380
+    print chr($_[0]) . chr($_[0]) . chr($_[0]) . "\n";  # 399
+    print chr($_[0]) . chr($_[0]) . "BB\n";       # 400
+    print chr($_[0]) . chr($_[0]) . "OO\n";       # 401
+    print chr($_[0]) . chr($_[0]) . "33\n";       # 402
+    print chr($_[0]) . chr($_[0]) . "..\n";       # 403
+    print chr($_[0]) . chr($_[0]) . "  \n";       # 404
+    print chr($_[0]) . chr($_[0]) . "様様\n";       # 405
+    print chr($_[0]) . chr($_[0]) . "クク\n";       # 406
+    print "B" . chr($_[0]) . chr($_[0]) . "B\n";  # 410
+    print "O" . chr($_[0]) . chr($_[0]) . "O\n";  # 411
+    print "3" . chr($_[0]) . chr($_[0]) . "3\n";  # 412
+    print "." . chr($_[0]) . chr($_[0]) . ".\n";  # 413
+    print " " . chr($_[0]) . chr($_[0]) . " \n";  # 414
+    print "様" . chr($_[0]) . chr($_[0]) . "様\n";  # 415
+    print "ク" . chr($_[0]) . chr($_[0]) . "ク\n";  # 416
+    print "BB" . chr($_[0]) . chr($_[0]) . "\n";  # 420
+    print "OO" . chr($_[0]) . chr($_[0]) . "\n";  # 421
+    print "33" . chr($_[0]) . chr($_[0]) . "\n";  # 422
+    print ".." . chr($_[0]) . chr($_[0]) . "\n";  # 423
+    print "  " . chr($_[0]) . chr($_[0]) . "\n";  # 424
+    print "様様" . chr($_[0]) . chr($_[0]) . "\n";  # 425
+    print "クク" . chr($_[0]) . chr($_[0]) . "\n";  # 426
+    print "3B" . chr($_[0]) . "B\n";                     # 480
+    print "3B-" . chr($_[0]) . "\n";                     # 481
+    print chr($_[0]) . chr($_[0]) . chr($_[0]) . chr($_[0]) . "\n";  # 499
+    print "BB" . chr($_[0]) . chr($_[0]) . "\t\n";   # 580
+    print "\tBB" . chr($_[0]) . chr($_[0]) . "\n";   # 581
+    print "BB-" . chr($_[0]) . chr($_[0]) . "\n";    # 582
+    print "🙂👍" . chr($_[0]) . "❤™\n";                # 583
+    print chr($_[0]) . chr($_[0]) . ".33\n";         # 584
+    print "3B-" . chr($_[0]) . "B\n";                # 585
+    print chr($_[0]) . chr($_[0]) . chr($_[0]) . chr($_[0]) . chr($_[0]) . "\n";  # 599
+  }
+  if(/<control>/){next}; # skip control characters
+  if($F[2] eq "Cs"){next}; # skip surrogates
+  if(/ First>/){$fi=hex("0x".$F[0]);next}; # generate blocks
+  if(/ Last>/){$la=hex("0x".$F[0]);for($fi..$la){pr($_)};next};
+  pr(hex("0x".$F[0])) # generate individual characters
+' UnicodeData.txt > _base-characters
diff --git a/postgres-appliance/tests/locales_test/test_locales.sh b/postgres-appliance/tests/locales_test/test_locales.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
+
+# shellcheck disable=SC1091
+source ../test_utils.sh
+
+TEST_CONTAINER_NAME='spilo-test'
+TEST_IMAGE=(
+    'registry.opensource.zalan.do/acid/spilo-cdp-14'
+    'spilo'
+)
+
+function main() {
+    for i in 0 1; do
+        rm_container "$TEST_CONTAINER_NAME"
+        docker run --rm -d --privileged \
+                --name "$TEST_CONTAINER_NAME" \
+                -v "$PWD":/home/postgres/tests \
+                -e SPILO_PROVIDER=local -e USE_OLD_LOCALES=true \
+                "${TEST_IMAGE[$i]}"  #USE_OLD_LOCALES takes no effect for cdp-14
+        attempts=0
+        while ! docker exec -i spilo-test su postgres -c "pg_isready"; do
+            if [[ "$attempts" -ge 15 ]]; then
+                docker logs "$TEST_CONTAINER_NAME"
+                exit 1
+            fi
+            ((attempts++))
+            sleep 1
+        done
+        /bin/bash -x ./generate_data.sh "$TEST_CONTAINER_NAME" "/home/postgres/output${i}.txt"
+        docker exec "$TEST_CONTAINER_NAME" mv "/home/postgres/output${i}.txt" "/home/postgres/tests"
+    done
+
+    diff -u output0.txt output1.txt > /dev/null || (echo "Outputs are different!" && exit 1)
+    rm -f output0.txt output1.txt
+}
+
+trap 'rm_container $TEST_CONTAINER_NAME' QUIT TERM EXIT
+
+main
diff --git a/postgres-appliance/tests/test_spilo.sh b/postgres-appliance/tests/test_spilo.sh
@@ -2,48 +2,13 @@
 
 cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 
-
-if ! docker info &> /dev/null; then
-    if podman info &> /dev/null; then
-        alias docker=podman
-        shopt -s expand_aliases
-    else
-        echo "docker/podman: command not found"
-        exit 1
-    fi
-fi
+# shellcheck disable=SC1091
+source ./test_utils.sh
 
 readonly PREFIX="demo-"
 readonly UPGRADE_SCRIPT="python3 /scripts/inplace_upgrade.py"
 readonly TIMEOUT=120
 
-if [[ -t 2 ]]; then
-    readonly RED="\033[1;31m"
-    readonly RESET="\033[0m"
-    readonly GREEN="\033[0;32m"
-else
-    readonly RED=""
-    readonly RESET=""
-    readonly GREEN=""
-fi
-
-function log_info() {
-    echo -e "${GREEN}$*${RESET}"
-}
-
-function log_error() {
-    echo -e "${RED}$*${RESET}"
-    exit 1
-}
-
-function start_containers() {
-    docker-compose up -d
-}
-
-function stop_containers() {
-    docker-compose rm -fs
-}
-
 function get_non_leader() {
     declare -r container=$1
 
@@ -54,11 +19,6 @@ function get_non_leader() {
     fi
 }
 
-function docker_exec() {
-    declare -r cmd=${*: -1:1}
-    docker exec "${@:1:$(($#-1))}" su postgres -c "$cmd"
-}
-
 function find_leader() {
     local container=$1
     local silent=$2
@@ -283,10 +243,6 @@ function verify_archive_mode_is_on() {
     [ "$archive_mode" = "on" ]
 }
 
-function run_test() {
-    "$@" || log_error "Test case $1 FAILED"
-    echo -e "Test case $1 ${GREEN}PASSED${RESET}"
-}
 
 # TEST SUITE 1 - In-place major upgrade 10->11->...->15
 # TEST SUITE 2 - Major upgrade 10->15 after wal-e clone

diff --git a/postgres-appliance/tests/test_utils.sh b/postgres-appliance/tests/test_utils.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+if ! docker info &> /dev/null; then
+    if podman info &> /dev/null; then
+        alias docker=podman
+        shopt -s expand_aliases
+    else
+        echo "docker/podman: command not found"
+        exit 1
+    fi
+fi
+
+set -a
+
+if [[ -t 2 ]]; then
+    readonly RED="\033[1;31m"
+    readonly RESET="\033[0m"
+    readonly GREEN="\033[0;32m"
+else
+    readonly RED=""
+    readonly RESET=""
+    readonly GREEN=""
+fi
+
+function log_info() {
+    echo -e "${GREEN}$*${RESET}"
+}
+
+function log_error() {
+    echo -e "${RED}$*${RESET}"
+    exit 1
+}
+
+function start_containers() {
+    docker-compose up -d
+}
+
+function stop_containers() {
+    docker-compose rm -fs
+}
+
+function rm_container() {
+    docker rm -f "$1"
+}
+
+function docker_exec() {
+    declare -r cmd=${*: -1:1}
+    docker exec "${@:1:$(($#-1))}" su postgres -c "$cmd"
+}
+
+function run_test() {
+    "$@" || log_error "Test case $1 FAILED"
+    echo -e "Test case $1 ${GREEN}PASSED${RESET}"
+}