From 43d4bde266cd0c42021202bf14d90b08177b8621 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Thu, 22 Feb 2024 05:51:57 +0100 Subject: [PATCH 1/6] apps: Rename unused file to fetch apps Signed-off-by: Mike Sul --- apps/{pull.py => fetch.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename apps/{pull.py => fetch.py} (100%) diff --git a/apps/pull.py b/apps/fetch.py similarity index 100% rename from apps/pull.py rename to apps/fetch.py From e6edcfebb6f614e4f0a570d8a1b8036131ae71d2 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Thu, 22 Feb 2024 06:42:01 +0100 Subject: [PATCH 2/6] apps: Add manual test for apps fetching Signed-off-by: Mike Sul --- apps/fetch.py | 11 +++++++---- tests/test_apps_fetch.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) create mode 100755 tests/test_apps_fetch.sh diff --git a/apps/fetch.py b/apps/fetch.py index 131ef3a2..adc29bba 100755 --- a/apps/fetch.py +++ b/apps/fetch.py @@ -18,7 +18,7 @@ def get_args(): 'store them on a file system') parser.add_argument('-f', '--factory', help='Apps Factory', required=True) parser.add_argument('-t', '--targets', help='Comma separated list of Targets to dump/fetch', required=True) - parser.add_argument('-a', '--token', help='Factory API Token, aka OSF Token', required=True) + parser.add_argument('-a', '--token-file', help='File where the Factory API Token is stored', required=True) parser.add_argument('-d', '--dst-dir', help='Directory to store apps and images in', required=True) parser.add_argument('-s', '--apps-shortlist', help='A coma separated list of Target Apps to fetch', default=None) @@ -30,11 +30,14 @@ def main(args: argparse.Namespace): exit_code = os.EX_OK try: target_list = args.targets.split(',') - factory_client = FactoryClient(args.factory, args.token) + with open(args.token_file) as f: + token = f.read() + factory_client = FactoryClient(args.factory, token) targets = factory_client.get_targets(target_list) - - apps_fetcher = SkopeAppFetcher(args.token, args.dst_dir) + if len(targets) == 0: + raise Exception(f"No targets found in the factory; factory: {args.factory}, {target_list}") + apps_fetcher = SkopeAppFetcher(token, args.dst_dir) for target in targets: target.shortlist = args.apps_shortlist apps_fetcher.fetch_target(target, force=True) diff --git a/tests/test_apps_fetch.sh b/tests/test_apps_fetch.sh new file mode 100755 index 00000000..4fc3a126 --- /dev/null +++ b/tests/test_apps_fetch.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright (c) 2024 Foundries.io +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +FACTORY=$1 +TOKEN_FILE=$2 +TARGETS=$3 +DST_DIR=$4 +APPS_SHORTLIST=$5 +API_HOST=${6-"api.foundries.io"} + +CMD=./apps/fetch.py +PARAMS="\ + --factory=${FACTORY} \ + --targets=${TARGETS} \ + --token-file=${TOKEN_FILE} \ + --dst-dir=/fetched-apps \ + --apps-shortlist=${APPS_SHORTLIST} +" + +docker run -v -it --rm \ + -e PYTHONPATH=. \ + -e H_RUN_URL="https://${API_HOST}" \ + -v "${PWD}":/ci-scripts \ + -v "${DST_DIR}":/fetched-apps \ + -v "${TOKEN_FILE}":"${TOKEN_FILE}" \ + -w /ci-scripts \ + foundries/lmp-image-tools ${CMD} ${PARAMS} \ No newline at end of file From 8b0634c49281392c2d056a1f47228ce5de75fb23 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Thu, 22 Feb 2024 07:00:31 +0100 Subject: [PATCH 3/6] apps: Fetch apps specified in targets file Signed-off-by: Mike Sul --- apps/fetch.py | 48 +++++++++++++++++++++++++------------ apps/target_apps_fetcher.py | 5 ---- tests/test_apps_fetch.sh | 20 +++++++++------- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/apps/fetch.py b/apps/fetch.py index adc29bba..9c9a409b 100755 --- a/apps/fetch.py +++ b/apps/fetch.py @@ -4,23 +4,43 @@ # SPDX-License-Identifier: Apache-2.0 import argparse +import json import logging import traceback import os import sys -from factory_client import FactoryClient from apps.target_apps_fetcher import SkopeAppFetcher +from factory_client import FactoryClient +from helpers import cmd + + +def fetch_target_apps(targets: dict, apps_shortlist: str, token: str, dst_dir: str): + apps_fetcher = SkopeAppFetcher(token, dst_dir) + for target_name, target_json in targets.items(): + apps_fetcher.fetch_target(FactoryClient.Target(target_name, target_json), + apps_shortlist, force=True) + + +def tar_fetched_apps(src_dir: str, out_file: str): + os.makedirs(os.path.dirname(out_file), exist_ok=True) + cmd('tar', '-cf', out_file, '-C', src_dir, '.') def get_args(): parser = argparse.ArgumentParser('Pull Targets Apps and their images from Registries and ' 'store them on a file system') parser.add_argument('-f', '--factory', help='Apps Factory', required=True) - parser.add_argument('-t', '--targets', help='Comma separated list of Targets to dump/fetch', required=True) - parser.add_argument('-a', '--token-file', help='File where the Factory API Token is stored', required=True) - parser.add_argument('-d', '--dst-dir', help='Directory to store apps and images in', required=True) - parser.add_argument('-s', '--apps-shortlist', help='A coma separated list of Target Apps to fetch', default=None) + parser.add_argument('-t', '--targets-file', + help='A json with Targets to dump/fetch apps for', required=True) + parser.add_argument('-a', '--token-file', + help='File where the Factory API Token is stored', required=True) + parser.add_argument('-d', '--fetch-dir', + help='Directory to fetch apps and images to', required=True) + parser.add_argument('-s', '--apps-shortlist', + help='Comma separated list of Target Apps to fetch', default=None) + parser.add_argument('-o', '--dst-dir', + help='Directory to output the tarred apps data to', required=True) args = parser.parse_args() return args @@ -29,18 +49,16 @@ def get_args(): def main(args: argparse.Namespace): exit_code = os.EX_OK try: - target_list = args.targets.split(',') with open(args.token_file) as f: token = f.read() - factory_client = FactoryClient(args.factory, token) + with open(args.targets_file) as f: + targets = json.load(f) - targets = factory_client.get_targets(target_list) - if len(targets) == 0: - raise Exception(f"No targets found in the factory; factory: {args.factory}, {target_list}") - apps_fetcher = SkopeAppFetcher(token, args.dst_dir) - for target in targets: - target.shortlist = args.apps_shortlist - apps_fetcher.fetch_target(target, force=True) + fetch_target_apps(targets, args.apps_shortlist, token, args.fetch_dir) + for target in targets.keys(): + out_file = os.path.join(args.dst_dir, f"{target}.apps.tar") + logging.info(f"Tarring fetched apps of {target} to {out_file}...") + tar_fetched_apps(os.path.join(args.fetch_dir, target), out_file) except Exception as exc: logging.error('Failed to pull Target apps and images: {}\n{}'.format(exc, traceback.format_exc())) @@ -49,7 +67,7 @@ def main(args: argparse.Namespace): if __name__ == '__main__': - logging.basicConfig(format='%(asctime)s %(levelname)s: Apps Puller: %(module)s: %(message)s', level=logging.INFO) + logging.basicConfig(format='%(asctime)s %(levelname)s: Apps Fetcher: %(module)s: %(message)s', level=logging.INFO) args = get_args() sys.exit(main(args)) diff --git a/apps/target_apps_fetcher.py b/apps/target_apps_fetcher.py index c7f1dbbc..9f633c37 100644 --- a/apps/target_apps_fetcher.py +++ b/apps/target_apps_fetcher.py @@ -49,11 +49,6 @@ def fetch_target(self, target: FactoryClient.Target, shortlist=None, force=False def fetch_target_apps(self, target: FactoryClient.Target, apps_shortlist=None, force=False): self.target_apps[target] = self._fetch_apps(target, apps_shortlist=apps_shortlist, force=force) - def fetch_apps(self, targets: dict, apps_shortlist=None): - for target_name, target_json in targets.items(): - target = FactoryClient.Target(target_name, target_json, shortlist=apps_shortlist) - self.target_apps[target] = self._fetch_apps(target, apps_shortlist=apps_shortlist) - def fetch_apps_images(self, graphdriver='overlay2', force=False): self._registry_client.login() for target, apps in self.target_apps.items(): diff --git a/tests/test_apps_fetch.sh b/tests/test_apps_fetch.sh index 4fc3a126..4ce138f8 100755 --- a/tests/test_apps_fetch.sh +++ b/tests/test_apps_fetch.sh @@ -5,25 +5,29 @@ set -euo pipefail FACTORY=$1 TOKEN_FILE=$2 -TARGETS=$3 -DST_DIR=$4 -APPS_SHORTLIST=$5 -API_HOST=${6-"api.foundries.io"} +TARGETS_FILE=$3 +FETCH_DIR=$4 +DST_DIR=$5 +APPS_SHORTLIST=${6-""} +API_HOST=${7-"api.foundries.io"} CMD=./apps/fetch.py PARAMS="\ --factory=${FACTORY} \ - --targets=${TARGETS} \ + --targets-file=${TARGETS_FILE} \ --token-file=${TOKEN_FILE} \ - --dst-dir=/fetched-apps \ - --apps-shortlist=${APPS_SHORTLIST} + --fetch-dir=/fetched-apps \ + --apps-shortlist=${APPS_SHORTLIST} \ + --dst-dir=/dst-dir \ " docker run -v -it --rm \ -e PYTHONPATH=. \ -e H_RUN_URL="https://${API_HOST}" \ -v "${PWD}":/ci-scripts \ - -v "${DST_DIR}":/fetched-apps \ + -v "${FETCH_DIR}":/fetched-apps \ + -v "${DST_DIR}":/dst-dir \ -v "${TOKEN_FILE}":"${TOKEN_FILE}" \ + -v "${TARGETS_FILE}":"${TARGETS_FILE}" \ -w /ci-scripts \ foundries/lmp-image-tools ${CMD} ${PARAMS} \ No newline at end of file From 92f19e8906bb3646b2f1d15a63a637288b3cd416 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Thu, 22 Feb 2024 14:12:56 +0100 Subject: [PATCH 4/6] apps: Fetch apps after publishing if enabled - Fetch apps of the new targets if enabled in the factory config. - Update TUF targets metadata with a link to the fecthed apps archive. Signed-off-by: Mike Sul --- apps/fetch.py | 19 ++++++++++++++++++- apps/publish.sh | 16 ++++++++++++++++ tests/test_apps_fetch.sh | 5 ++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/apps/fetch.py b/apps/fetch.py index 9c9a409b..3bd5bdf9 100755 --- a/apps/fetch.py +++ b/apps/fetch.py @@ -41,6 +41,8 @@ def get_args(): help='Comma separated list of Target Apps to fetch', default=None) parser.add_argument('-o', '--dst-dir', help='Directory to output the tarred apps data to', required=True) + parser.add_argument('-tt', '--tuf-targets', + help='TUF targets to be updated with URI to fetched app archive') args = parser.parse_args() return args @@ -55,10 +57,25 @@ def main(args: argparse.Namespace): targets = json.load(f) fetch_target_apps(targets, args.apps_shortlist, token, args.fetch_dir) - for target in targets.keys(): + for target, target_json in targets.items(): out_file = os.path.join(args.dst_dir, f"{target}.apps.tar") logging.info(f"Tarring fetched apps of {target} to {out_file}...") tar_fetched_apps(os.path.join(args.fetch_dir, target), out_file) + target_json["custom"]["fetched-apps"] = { + "uri": os.path.join(os.environ["H_RUN_URL"], f"{target}.apps.tar"), + "shortlist": args.apps_shortlist, + } + with open(args.targets_file, "w") as f: + json.dump(targets, f) + + if args.tuf_targets: + with open(args.tuf_targets, "r") as f: + tuf_targets = json.load(f) + + tuf_targets["targets"].update(targets) + + with open(args.tuf_targets, "w") as f: + json.dump(tuf_targets, f) except Exception as exc: logging.error('Failed to pull Target apps and images: {}\n{}'.format(exc, traceback.format_exc())) diff --git a/apps/publish.sh b/apps/publish.sh index 6e6f0e16..6996d97c 100755 --- a/apps/publish.sh +++ b/apps/publish.sh @@ -33,6 +33,10 @@ PLATFORMS=${MANIFEST_PLATFORMS_DEFAULT-""} TUF_TARGETS_EXPIRE=${TUF_TARGETS_EXPIRE-1Y} +FETCH_APPS=${FETCH_APPS-""} +FETCH_APPS_DIR="${FETCH_APPS_DIR-$(mktemp -u -d -p /var/cache/apps)}" +FETCH_APPS_SHORTLIST="${FETCH_APPS_SHORTLIST-""}" + require_params FACTORY ARCHIVE TARGET_TAG TUF_TARGETS_EXPIRE #-- END: Input params @@ -91,6 +95,18 @@ status "Publishing apps; version: ${APPS_VERSION}, Target tag: ${TARGET_TAG}" --target-version="${TARGET_VERSION}" \ --new-targets-file="${ARCHIVE}/targets-created.json" +if [ "${FETCH_APPS}" == "1" ]; then + status "Fetching and archiving apps..." + "${HERE}/fetch.py" \ + --factory "${FACTORY}" \ + --targets-file="${ARCHIVE}/targets-created.json" \ + --token-file="${SECRETS}/osftok" \ + --fetch-dir="${FETCH_APPS_DIR}" \ + --apps-shortlist="${FETCH_APPS_SHORTLIST}" \ + --dst-dir="${ARCHIVE}" \ + --tuf-targets="${TUF_REPO}/roles/unsigned/targets.json" +fi + cp "${TUF_REPO}/roles/unsigned/targets.json" "${ARCHIVE}/targets-after.json" echo "Signing local TUF targets" diff --git a/tests/test_apps_fetch.sh b/tests/test_apps_fetch.sh index 4ce138f8..3727c4d6 100755 --- a/tests/test_apps_fetch.sh +++ b/tests/test_apps_fetch.sh @@ -9,7 +9,8 @@ TARGETS_FILE=$3 FETCH_DIR=$4 DST_DIR=$5 APPS_SHORTLIST=${6-""} -API_HOST=${7-"api.foundries.io"} +TUF_TARGETS_FILE=$7 +API_HOST=${8-"api.foundries.io"} CMD=./apps/fetch.py PARAMS="\ @@ -19,6 +20,7 @@ PARAMS="\ --fetch-dir=/fetched-apps \ --apps-shortlist=${APPS_SHORTLIST} \ --dst-dir=/dst-dir \ + --tuf-targets=${TUF_TARGETS_FILE} \ " docker run -v -it --rm \ @@ -29,5 +31,6 @@ docker run -v -it --rm \ -v "${DST_DIR}":/dst-dir \ -v "${TOKEN_FILE}":"${TOKEN_FILE}" \ -v "${TARGETS_FILE}":"${TARGETS_FILE}" \ + -v "${TUF_TARGETS_FILE}":"${TUF_TARGETS_FILE}" \ -w /ci-scripts \ foundries/lmp-image-tools ${CMD} ${PARAMS} \ No newline at end of file From 43aad96e9dc688e6fe6546a28d057444f30dcdeb Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Tue, 27 Feb 2024 18:15:29 +0100 Subject: [PATCH 5/6] assemble: Fetch app archive if exists Fetch the app archive from the publish run if 1) it exists, 2. apps in the archive is sub-set of the apps required for preloading. Signed-off-by: Mike Sul --- assemble.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/assemble.py b/assemble.py index 5b1d08de..5887b127 100755 --- a/assemble.py +++ b/assemble.py @@ -7,15 +7,23 @@ import os import json import argparse +import io import logging import shutil +import tarfile import requests from math import ceil from time import sleep from typing import NamedTuple -from helpers import cmd, Progress +from helpers import ( + cmd, + http_get, + Progress, + status, +) + from apps.target_apps_fetcher import TargetAppsFetcher, SkopeAppFetcher from factory_client import FactoryClient @@ -265,6 +273,51 @@ def fetch_restorable_apps(target: FactoryClient.Target, dst_dir: str, shortlist: return AppsDesc(apps_fetcher.target_dir(target.name), apps_fetcher.get_target_apps_size(target)) +def check_and_get_fetched_apps_uri(target: FactoryClient.Target, shortlist: [str] = None): + fetched_apps = None + fetched_apps_uri = None + if "fetched-apps" in target.json.get("custom", {}): + fetched_apps_str = target.json["custom"]["fetched-apps"].get("shortlist") + if fetched_apps_str: + fetched_apps = set( + x.strip() for x in fetched_apps_str.split(',') if x) + else: + # if `shortlist` is not defined or empty then all target apps were fetched + fetched_apps = set(target.apps().keys()) + + apps_to_fetch = set(shortlist) if shortlist else set(target.apps().keys()) + + if fetched_apps.issubset(apps_to_fetch): + # if the previously fetched apps is a sub-set of the apps to be fetched then + # enable getting and reusing the previously fetched apps + fetched_apps_uri = target.json["custom"]["fetched-apps"]["uri"] + + return fetched_apps_uri, fetched_apps + + +def get_and_extract_fetched_apps(uri: str, token: str, out_dir: str): + resp = http_get(uri, headers={ + "OSF-TOKEN": token, + "Connection": "keep-alive", + "Keep-Alive": "timeout=1200, max=1" + # keep connection alive for 1 request for 20m + }, stream=True) + + total_length = int(resp.headers["content-length"]) + progress_percent = 5 + progress_step = total_length * (progress_percent / 100) + + last_reported_pos = 0 + with io.BufferedReader(resp.raw, buffer_size=1024 * 1024) as buf_reader: + with tarfile.open(fileobj=buf_reader, mode="r|") as ts: + for m in ts: + ts.extract(m, out_dir) + if buf_reader.tell() - last_reported_pos > progress_step: + percent = round(buf_reader.tell() / total_length * 100) + status("Downloaded %d%% " % percent, with_ts=True) + last_reported_pos = buf_reader.tell() + + def archive_and_output_assembled_wic(wic_image: str, out_image_dir: str): logger.info('Gzip and move resultant system image to the specified destination folder: {}'.format(out_image_dir)) subprocess.check_call(['bmaptool', 'create', wic_image, '-o', wic_image + '.bmap']) @@ -336,6 +389,15 @@ def get_args(): target.lmp_version = release_info.lmp_version if args.app_type == 'restorable' or (not args.app_type and release_info.lmp_version > 84): logger.info('Fetching Restorable Apps...') + previously_fetched_apps_uri, previously_fetched_apps \ + = check_and_get_fetched_apps_uri(target, args.app_shortlist) + if previously_fetched_apps_uri: + target_apps_dir = os.path.join(apps_root_dir, target.name) + logger.info("Fetching the app archive from the publish run; uri:" + f" {previously_fetched_apps_uri}, apps: {previously_fetched_apps}") + get_and_extract_fetched_apps(previously_fetched_apps_uri, args.token, + target_apps_dir) + logger.info(f"The fetched app archive is extracted to {target_apps_dir}") apps_desc = fetch_restorable_apps(target, apps_root_dir, args.app_shortlist, args.token) fetched_apps[target.name] = (apps_desc, os.path.join(args.out_image_dir, target.tags[0])) fetch_progress.tick() From fdbf8a26df82bafce775308b531902fc651a6901 Mon Sep 17 00:00:00 2001 From: Mike Sul Date: Thu, 29 Feb 2024 11:38:32 +0100 Subject: [PATCH 6/6] customize-target: Copy fetched apps if present Copy the fetched apps value from the target json that the target being built is based on, if present. Signed-off-by: Mike Sul --- lmp/customize-target.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lmp/customize-target.py b/lmp/customize-target.py index 2a65a9c4..7b26bb98 100755 --- a/lmp/customize-target.py +++ b/lmp/customize-target.py @@ -131,6 +131,9 @@ def merge(targets_json, target_name, lmp_manifest_sha, arch, image_name, u['tgt']['custom']['origUriApps'] = apps_uri else: u['tgt']['custom']['origUriApps'] = u['prev']['custom']['uri'] + fetched_apps = u['prev']['custom'].get('fetched-apps') + if fetched_apps: + u['tgt']['custom']['fetched-apps'] = fetched_apps changed = True if changed: