Fetch apps for offline update

apps/fetch.py

@@ -0,0 +1,90 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2021 Foundries.io
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import json
+import logging
+import traceback
+import os
+import sys
+
+from apps.target_apps_fetcher import SkopeAppFetcher
+from factory_client import FactoryClient
+from helpers import cmd
+
+
+def fetch_target_apps(targets: dict, apps_shortlist: str, token: str, dst_dir: str):
+    apps_fetcher = SkopeAppFetcher(token, dst_dir)
+    for target_name, target_json in targets.items():
+        apps_fetcher.fetch_target(FactoryClient.Target(target_name, target_json),
+                                  apps_shortlist, force=True)
+
+
+def tar_fetched_apps(src_dir: str, out_file: str):
+    os.makedirs(os.path.dirname(out_file), exist_ok=True)
+    cmd('tar', '-cf', out_file, '-C', src_dir, '.')
+
+
+def get_args():
+    parser = argparse.ArgumentParser('Pull Targets Apps and their images from Registries and '
+                                     'store them on a file system')
+    parser.add_argument('-f', '--factory', help='Apps Factory', required=True)
+    parser.add_argument('-t', '--targets-file',
+                        help='A json with Targets to dump/fetch apps for', required=True)
+    parser.add_argument('-a', '--token-file',
+                        help='File where the Factory API Token is stored', required=True)
+    parser.add_argument('-d', '--fetch-dir',
+                        help='Directory to fetch apps and images to', required=True)
+    parser.add_argument('-s', '--apps-shortlist',
+                        help='Comma separated list of Target Apps to fetch', default=None)
+    parser.add_argument('-o', '--dst-dir',
+                        help='Directory to output the tarred apps data to', required=True)
+    parser.add_argument('-tt', '--tuf-targets',
+                        help='TUF targets to be updated with URI to fetched app archive')
+
+    args = parser.parse_args()
+    return args
+
+
+def main(args: argparse.Namespace):
+    exit_code = os.EX_OK
+    try:
+        with open(args.token_file) as f:
+            token = f.read()
+        with open(args.targets_file) as f:
+            targets = json.load(f)
+
+        fetch_target_apps(targets, args.apps_shortlist, token, args.fetch_dir)
+        for target, target_json in targets.items():
+            out_file = os.path.join(args.dst_dir, f"{target}.apps.tar")
+            logging.info(f"Tarring fetched apps of {target} to {out_file}...")
+            tar_fetched_apps(os.path.join(args.fetch_dir, target), out_file)
+            target_json["custom"]["fetched-apps"] = {
+                "uri": os.path.join(os.environ["H_RUN_URL"], f"{target}.apps.tar"),
+                "shortlist": args.apps_shortlist,
+            }
+        with open(args.targets_file, "w") as f:
+            json.dump(targets, f)
+
+        if args.tuf_targets:
+            with open(args.tuf_targets, "r") as f:
+                tuf_targets = json.load(f)
+
+            tuf_targets["targets"].update(targets)
+
+            with open(args.tuf_targets, "w") as f:
+                json.dump(tuf_targets, f)
+
+    except Exception as exc:
+        logging.error('Failed to pull Target apps and images: {}\n{}'.format(exc, traceback.format_exc()))
+        exit_code = os.EX_SOFTWARE
+    return exit_code
+
+
+if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)s %(levelname)s: Apps Fetcher: %(module)s: %(message)s', level=logging.INFO)
+    args = get_args()
+    sys.exit(main(args))
+

apps/publish.sh

@@ -33,6 +33,10 @@ PLATFORMS=${MANIFEST_PLATFORMS_DEFAULT-""}
 
 TUF_TARGETS_EXPIRE=${TUF_TARGETS_EXPIRE-1Y}
 
+FETCH_APPS=${FETCH_APPS-""}
+FETCH_APPS_DIR="${FETCH_APPS_DIR-$(mktemp -u -d -p /var/cache/apps)}"
+FETCH_APPS_SHORTLIST="${FETCH_APPS_SHORTLIST-""}"
+
 require_params FACTORY ARCHIVE TARGET_TAG TUF_TARGETS_EXPIRE
 #-- END: Input params
 
@@ -91,6 +95,18 @@ status "Publishing apps; version: ${APPS_VERSION}, Target tag: ${TARGET_TAG}"
     --target-version="${TARGET_VERSION}" \
     --new-targets-file="${ARCHIVE}/targets-created.json"
 
+if [ "${FETCH_APPS}" == "1" ]; then
+  status "Fetching and archiving apps..."
+  "${HERE}/fetch.py" \
+      --factory "${FACTORY}" \
+      --targets-file="${ARCHIVE}/targets-created.json" \
+      --token-file="${SECRETS}/osftok" \
+      --fetch-dir="${FETCH_APPS_DIR}" \
+      --apps-shortlist="${FETCH_APPS_SHORTLIST}" \
+      --dst-dir="${ARCHIVE}" \
+      --tuf-targets="${TUF_REPO}/roles/unsigned/targets.json"
+fi
+
 cp "${TUF_REPO}/roles/unsigned/targets.json" "${ARCHIVE}/targets-after.json"
 
 echo "Signing local TUF targets"

apps/target_apps_fetcher.py

@@ -49,11 +49,6 @@ def fetch_target(self, target: FactoryClient.Target, shortlist=None, force=False
     def fetch_target_apps(self, target: FactoryClient.Target, apps_shortlist=None, force=False):
         self.target_apps[target] = self._fetch_apps(target, apps_shortlist=apps_shortlist, force=force)
 
-    def fetch_apps(self, targets: dict, apps_shortlist=None):
-        for target_name, target_json in targets.items():
-            target = FactoryClient.Target(target_name, target_json, shortlist=apps_shortlist)
-            self.target_apps[target] = self._fetch_apps(target, apps_shortlist=apps_shortlist)
-
     def fetch_apps_images(self, graphdriver='overlay2', force=False):
         self._registry_client.login()
         for target, apps in self.target_apps.items():

assemble.py

@@ -7,15 +7,23 @@
 import os
 import json
 import argparse
+import io
 import logging
 import shutil
+import tarfile
 
 import requests
 from math import ceil
 from time import sleep
 from typing import NamedTuple
 
-from helpers import cmd, Progress
+from helpers import (
+    cmd,
+    http_get,
+    Progress,
+    status,
+)
+
 from apps.target_apps_fetcher import TargetAppsFetcher, SkopeAppFetcher
 from factory_client import FactoryClient
 
@@ -265,6 +273,51 @@ def fetch_restorable_apps(target: FactoryClient.Target, dst_dir: str, shortlist:
     return AppsDesc(apps_fetcher.target_dir(target.name), apps_fetcher.get_target_apps_size(target))
 
 
+def check_and_get_fetched_apps_uri(target: FactoryClient.Target, shortlist: [str] = None):
+    fetched_apps = None
+    fetched_apps_uri = None
+    if "fetched-apps" in target.json.get("custom", {}):
+        fetched_apps_str = target.json["custom"]["fetched-apps"].get("shortlist")
+        if fetched_apps_str:
+            fetched_apps = set(
+                x.strip() for x in fetched_apps_str.split(',') if x)
+        else:
+            # if `shortlist` is not defined or empty then all target apps were fetched
+            fetched_apps = set(target.apps().keys())
+
+        apps_to_fetch = set(shortlist) if shortlist else set(target.apps().keys())
+
+        if fetched_apps.issubset(apps_to_fetch):
+            # if the previously fetched apps is a sub-set of the apps to be fetched then
+            # enable getting and reusing the previously fetched apps
+            fetched_apps_uri = target.json["custom"]["fetched-apps"]["uri"]
+
+    return fetched_apps_uri, fetched_apps
+
+
+def get_and_extract_fetched_apps(uri: str, token: str, out_dir: str):
+    resp = http_get(uri, headers={
+        "OSF-TOKEN": token,
+        "Connection": "keep-alive",
+        "Keep-Alive": "timeout=1200, max=1"
+        # keep connection alive for 1 request for 20m
+    }, stream=True)
+
+    total_length = int(resp.headers["content-length"])
+    progress_percent = 5
+    progress_step = total_length * (progress_percent / 100)
+
+    last_reported_pos = 0
+    with io.BufferedReader(resp.raw, buffer_size=1024 * 1024) as buf_reader:
+        with tarfile.open(fileobj=buf_reader, mode="r|") as ts:
+            for m in ts:
+                ts.extract(m, out_dir)
+                if buf_reader.tell() - last_reported_pos > progress_step:
+                    percent = round(buf_reader.tell() / total_length * 100)
+                    status("Downloaded %d%% " % percent, with_ts=True)
+                    last_reported_pos = buf_reader.tell()
+
+
 def archive_and_output_assembled_wic(wic_image: str, out_image_dir: str):
     logger.info('Gzip and move resultant system image to the specified destination folder: {}'.format(out_image_dir))
     subprocess.check_call(['bmaptool', 'create', wic_image, '-o', wic_image + '.bmap'])
@@ -336,6 +389,15 @@ def get_args():
             target.lmp_version = release_info.lmp_version
             if args.app_type == 'restorable' or (not args.app_type and release_info.lmp_version > 84):
                 logger.info('Fetching Restorable Apps...')
+                previously_fetched_apps_uri, previously_fetched_apps \
+                    = check_and_get_fetched_apps_uri(target, args.app_shortlist)
+                if previously_fetched_apps_uri:
+                    target_apps_dir = os.path.join(apps_root_dir, target.name)
+                    logger.info("Fetching the app archive from the publish run; uri:"
+                                f" {previously_fetched_apps_uri}, apps: {previously_fetched_apps}")
+                    get_and_extract_fetched_apps(previously_fetched_apps_uri, args.token,
+                                                 target_apps_dir)
+                    logger.info(f"The fetched app archive is extracted to {target_apps_dir}")
                 apps_desc = fetch_restorable_apps(target, apps_root_dir, args.app_shortlist, args.token)
                 fetched_apps[target.name] = (apps_desc, os.path.join(args.out_image_dir, target.tags[0]))
             fetch_progress.tick()

lmp/customize-target.py

@@ -131,6 +131,9 @@ def merge(targets_json, target_name, lmp_manifest_sha, arch, image_name,
                     u['tgt']['custom']['origUriApps'] = apps_uri
                 else:
                     u['tgt']['custom']['origUriApps'] = u['prev']['custom']['uri']
+                fetched_apps = u['prev']['custom'].get('fetched-apps')
+                if fetched_apps:
+                    u['tgt']['custom']['fetched-apps'] = fetched_apps
                 changed = True
 
     if changed:

tests/test_apps_fetch.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright (c) 2024 Foundries.io
+# SPDX-License-Identifier: Apache-2.0
+set -euo pipefail
+
+FACTORY=$1
+TOKEN_FILE=$2
+TARGETS_FILE=$3
+FETCH_DIR=$4
+DST_DIR=$5
+APPS_SHORTLIST=${6-""}
+TUF_TARGETS_FILE=$7
+API_HOST=${8-"api.foundries.io"}
+
+CMD=./apps/fetch.py
+PARAMS="\
+  --factory=${FACTORY} \
+  --targets-file=${TARGETS_FILE} \
+  --token-file=${TOKEN_FILE} \
+  --fetch-dir=/fetched-apps \
+  --apps-shortlist=${APPS_SHORTLIST} \
+  --dst-dir=/dst-dir \
+  --tuf-targets=${TUF_TARGETS_FILE} \
+"
+
+docker run -v -it --rm \
+  -e PYTHONPATH=. \
+  -e H_RUN_URL="https://${API_HOST}" \
+  -v "${PWD}":/ci-scripts \
+  -v "${FETCH_DIR}":/fetched-apps \
+  -v "${DST_DIR}":/dst-dir \
+  -v "${TOKEN_FILE}":"${TOKEN_FILE}" \
+  -v "${TARGETS_FILE}":"${TARGETS_FILE}" \
+  -v "${TUF_TARGETS_FILE}":"${TUF_TARGETS_FILE}" \
+  -w /ci-scripts \
+  foundries/lmp-image-tools ${CMD} ${PARAMS}