Merge remote-tracking branch 'origin/development' into feature/enable…

…-postgres-query-logs

README.md

@@ -6,7 +6,7 @@
 - Docker compose >= 1.17
 - RAM memory: At least 4Gb for instance, preferrably 8Gb.
 
-On Ubuntu 18.04:
+On Ubuntu 22.04:
 
 ```
 $ sudo apt install docker.io docker-compose python3 python3-setuptools
@@ -91,6 +91,7 @@ Some notes:
 - Use option `--run-scripts=DIRECTORY` to run shell scripts (.sh) from a directory within the `dhis2-core` container. By default, a script is run **after** postgres starts (`host=db`, `port=5432`) but **before** Tomcat starts; if its filename starts with prefix "post", it will be run **after** Tomcat is available. `curl` and typical shell tools are available on that Alpine Linux environment. Note that the Dhis2 endpoint is always `http://localhost:8080/${deployPath}`, regardless of the public port that the instance is exposed to.
 - Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`)
 - Use option `--postgis-version=13-3.1-alpine` to specify the PostGIS version to use. By default, 10-2.5-alpine is used.
+- Use option `--debug-port=PORT` to specify the debug port of the Tomcat process.
 
 #### Custom DHIS2 dhis.conf
 

requirements.txt

@@ -2,3 +2,5 @@ Flask==2.0.3
 python-dotenv
 Flask_Cors==3.0.10
 Werkzeug==2.3.7
+requests==2.31.0
+PyYAML

src/d2_docker/commands/start.py

@@ -26,7 +26,8 @@ def setup(parser):
     parser.add_argument("--tomcat-server-xml", metavar="FILE", help=server_xml_help)
     parser.add_argument("--dhis-conf", metavar="FILE", help=dhis_conf_help)
     parser.add_argument("--run-sql", metavar="DIRECTORY", help="Run .sql[.gz] files in directory")
-    parser.add_argument("--db-port", metavar="PORT", help="Export DB Postgres port")
+    parser.add_argument("--debug-port", metavar="PORT", help="Expose DHIS2 core debug port")
+    parser.add_argument("--db-port", metavar="PORT", help="Expose DB Postgres port")
     parser.add_argument(
         "--run-scripts",
         metavar="DIRECTORY",
@@ -102,6 +103,7 @@ def start(args):
             core_image=core_image,
             load_from_data=override_containers,
             post_sql_dir=args.run_sql,
+            debug_port=args.debug_port,
             db_port=args.db_port,
             scripts_dir=args.run_scripts,
             deploy_path=deploy_path,

src/d2_docker/config/dhis2-core-entrypoint.sh

@@ -6,19 +6,51 @@
 # We need our custom entrypoint to perform the following different tasks:
 #  - Make files in TOMCATDIR be of user tomcat (so we can change tomcat files in pre/post scripts)
 #
-set -e  # exit on errors
+set -e # exit on errors
 
 WARFILE=/usr/local/tomcat/webapps/ROOT.war
 TOMCATDIR=/usr/local/tomcat
 DHIS2HOME=/DHIS2_home
 DATA_DIR=/data
 
+debug() {
+    echo "[dhis2-core-entrypoint] $*" >&2
+}
+
+curl_status_connection_refused=7
+
+wait_for_data_container_to_finish_copy() {
+    # We must wait for the data container before starting Tomcat. As ping is not installed
+    # in the image, use curl instead.
+    local statuscode
+
+    while true; do
+        statuscode=0
+        curl -sS --connect-timeout 1 data || statuscode=$?
+
+        case $statuscode in
+        "$curl_status_connection_refused")
+            debug "data container is still running, wait"
+            sleep 2
+            continue
+            ;;
+        *)
+            debug "data container has finished, continue"
+            break
+            ;;
+        esac
+    done
+
+}
+
 if [ "$(id -u)" = "0" ]; then
     if [ -f $WARFILE ]; then
         unzip -q $WARFILE -d $TOMCATDIR/webapps/ROOT
-        rm -v $WARFILE  # just to save space
+        rm -v $WARFILE # just to save space
     fi
 
+    wait_for_data_container_to_finish_copy
+
     mkdir -p $DATA_DIR/apps
     chown -R tomcat:tomcat $TOMCATDIR $DATA_DIR/apps $DHIS2HOME
     chmod -R u=rwX,g=rX,o-rwx $TOMCATDIR $DATA_DIR/apps $DHIS2HOME

src/d2_docker/config/nginx.conf

@@ -40,7 +40,6 @@ http {
       proxy_set_header Host $http_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_cookie_path / "/; secure; HttpOnly; SameSite=strict";
       proxy_pass http://core;
     }
 

src/d2_docker/docker-compose.yml

@@ -15,7 +15,7 @@ services:
             - "/etc/timezone:/etc/timezone:ro"
             - "/etc/localtime:/etc/localtime:ro"
         environment:
-            CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH}"
+            CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH} -Xdebug -Xrunjdwp:transport=dt_socket,address=0.0.0.0:8000,server=y,suspend=n"
             JAVA_OPTS: "-Xmx7500m -Xms4000m ${JAVA_OPTS}"
             LOAD_FROM_DATA: "${LOAD_FROM_DATA}"
             DEPLOY_PATH: "${DEPLOY_PATH}"
@@ -25,6 +25,9 @@ services:
         restart: unless-stopped
         depends_on:
             - "db"
+            - "data"
+        ports:
+            - "${DHIS2_CORE_DEBUG_PORT}"
     db:
         image: "postgis/postgis:${POSTGIS_VERSION:-14-3.2-alpine}"
         shm_size: 1gb

src/d2_docker/utils.py

@@ -1,3 +1,4 @@
+import atexit
 import contextlib
 import subprocess
 import logging
@@ -7,6 +8,7 @@
 import socket
 import tempfile
 import time
+import yaml
 import urllib.request
 from distutils import dir_util
 from pathlib import Path
@@ -18,11 +20,12 @@
 DHIS2_DATA_IMAGE = "dhis2-data"
 IMAGE_NAME_LABEL = "com.eyeseetea.image-name"
 DOCKER_COMPOSE_SERVICES = ["gateway", "core", "db"]
-ROOT_PATH = os.environ.get("ROOT_PATH")
+PROJECT_DIR = os.path.dirname(os.path.realpath(__file__))
+ROOT_PATH = os.environ.get("ROOT_PATH") or PROJECT_DIR
 
-
-def get_dhis2_war(version):
-    match = re.match(r"^(\d+.\d+)", version)
+def get_dhis2_war_url(version):
+    match = (re.match(r"^(\d+.\d+)", version) if version.startswith("2.")
+        else re.match(r"^(\d+)", version))
     if not match:
         raise D2DockerError("Invalid version: {}".format(version))
     short_version = match[1]
@@ -246,6 +249,7 @@ def run_docker_compose(
     port=None,
     load_from_data=True,
     post_sql_dir=None,
+    debug_port=None,
     db_port=None,
     bind_ip=None,
     scripts_dir=None,
@@ -274,6 +278,7 @@ def run_docker_compose(
     env_pairs = [
         ("DHIS2_DATA_IMAGE", final_image_name),
         ("DHIS2_CORE_PORT", str(port)) if port else None,
+        ("DHIS2_CORE_DEBUG_PORT", "{}:8000".format(debug_port)) if debug_port else None,
         ("DHIS2_CORE_IP", bind_ip + ":") if bind_ip else "",
         ("DHIS2_CORE_IMAGE", core_image_name),
         ("LOAD_FROM_DATA", "yes" if load_from_data else "no"),
@@ -288,14 +293,39 @@ def run_docker_compose(
         ("POSTGIS_VERSION", postgis_version),
         ("DB_PORT", ("{}:5432".format(db_port) if db_port else "0:1000")),
         # Add ROOT_PATH from environment (required when run inside a docker)
-        ("ROOT_PATH", ROOT_PATH or "."),
+        ("ROOT_PATH", ROOT_PATH),
         ("PSQL_ENABLE_QUERY_LOGS", "") if not enable_postgres_queries_logging else None,
     ]
     env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None)
 
-    yaml_path = os.path.join(os.path.dirname(__file__), "docker-compose.yml")
-    return run(["docker-compose", "-f", yaml_path, "-p", project_name, *args], env=env, **kwargs)
+    def process_yaml(data):
+        if "DHIS2_CORE_DEBUG_PORT" not in env:
+            core = data["services"]["core"]
+            core["ports"] = [port for port in core["ports"] if "DHIS2_CORE_DEBUG_PORT" not in port]
+
+        return data
+
+    temp_compose = build_docker_compose(process_yaml)
+
+    return run(["docker-compose", "-f", temp_compose.name, "-p", project_name, *args], env=env, **kwargs)
+
+
+def build_docker_compose(process_yaml):
+    docker_compose_path = os.path.join(ROOT_PATH, "docker-compose.yml")
+
+    with open(docker_compose_path, 'r') as file:
+        data = yaml.safe_load(file)
+
+    data_processed = process_yaml(data)
+
+    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.yml') as temp_compose:
+        yaml_contents = yaml.dump(data_processed)
+        temp_compose.write(yaml_contents)
+
+    atexit.register(lambda: os.remove(temp_compose.name))
+    logger.debug("Docker compose file: {}".format(temp_compose.name))
 
+    return temp_compose
 
 def get_config_path(default_filename, path):
     return os.path.abspath(path) if path else get_config_file(default_filename)
@@ -304,7 +334,7 @@ def get_config_path(default_filename, path):
 def get_absdir_for_docker_volume(directory):
     """Return absolute path for given directory, with fallback to empty directory."""
     if not directory:
-        empty_directory = os.path.join(ROOT_PATH or os.path.dirname(__file__), ".empty")
+        empty_directory = os.path.join(ROOT_PATH, ".empty")
         return empty_directory
     elif not Path(directory).is_dir():
         raise D2DockerError("Should be a directory: {}".format(directory))
@@ -315,7 +345,7 @@ def get_absdir_for_docker_volume(directory):
 def get_absfile_for_docker_volume(file_path):
     """Return absolute path for given file, with fallback to empty file."""
     if not file_path:
-        return os.path.join(ROOT_PATH or os.path.dirname(__file__), ".empty", "placeholder")
+        return os.path.join(ROOT_PATH, ".empty", "placeholder")
     else:
         return os.path.abspath(file_path)
 
@@ -341,9 +371,8 @@ def get_item_type(name):
 
 def get_docker_directory(image_type, args=None):
     """Return docker directory for dhis2-data."""
-    script_dir = os.path.dirname(os.path.realpath(__file__))
     subdir = "images/dhis2-core" if image_type == "core" else "images/dhis2-data"
-    basedir = args and args.dhis2_docker_images_directory or script_dir
+    basedir = args and args.dhis2_docker_images_directory or PROJECT_DIR
     docker_dir = os.path.join(basedir, subdir)
 
     if not Path(docker_dir).is_dir():
@@ -548,7 +577,7 @@ def create_core(
             logger.debug("Copy WAR file: {} -> {}".format(war, war_path))
             shutil.copy(war, war_path)
         elif version:
-            war_url = get_dhis2_war(version)
+            war_url = get_dhis2_war_url(version)
             logger.info("Download file: {}".format(war_url))
             urllib.request.urlretrieve(war_url, war_path)  # nosec
         else: