From 9c3d7ce1496d2835f2f94aea9e7bc2d128c96f08 Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 6 Oct 2024 11:28:55 +0100 Subject: [PATCH] Deploy artifacts to cdn (#2) * feat: upload dependencies and generate bootstrap in ci * fix: use correct AWS key env vars * feat: slim down bootstrap.json --- .github/workflows/deploy.yml | 33 ++++++++++- bootstrap-base.json | 1 + bootstrap.py | 106 +++++++++++++++++++++++++++++++++++ deploy-dependencies.py | 71 +++++++++++++++++++++++ pom.xml | 35 +----------- settings.xml | 6 +- 6 files changed, 212 insertions(+), 40 deletions(-) create mode 100644 bootstrap-base.json create mode 100644 bootstrap.py create mode 100644 deploy-dependencies.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7082e842..d4569723 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,12 +10,14 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.AWS_S3_ACCESS_KEY }} - aws-secret-access-key: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: eu-west-1 + - uses: actions/checkout@v4 with: submodules: 'true' + - name: cache uses: actions/cache@v4 with: @@ -24,10 +26,35 @@ jobs: key: ${{ runner.os }}-cache-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-cache- + - name: Set up JDK 11 uses: actions/setup-java@v4 with: java-version: 11 distribution: temurin + - name: build - run: mvn deploy \ No newline at end of file + run: mvn deploy -DskipTests=true -DuniqueVersion=false --settings settings.xml + + - name: Gather dependencies + run: mvn -Dmdep.copyPom=true dependency:copy-dependencies && mvn dependency:list -DoutputFile=./deps.txt -Dsort=true -DincludeScope=compile -DincludeScope=runtime -DincludeTypes=jar + + - name: Deploy dependencies + run: python ./deploy-dependencies.py + + - name: Generate bootstrap + run: python ./bootstrap.py > bootstrap.json + + - name: Generate bootstrap signature + run: 'echo "$SIGNING_PRIVATE_KEY" > private.pem && openssl dgst -sha256 -sign private.pem -out bootstrap.json.sha256 bootstrap.json' + env: + SIGNING_PRIVATE_KEY: ${{ secrets.SIGNING_PRIVATE_KEY }} + + - name: Upload bootstrap + run: 'aws s3 cp bootstrap.json "s3://cdn.rsprox.net/runelite/launcher/" && aws s3 cp bootstrap.json.sha256 "s3://cdn.rsprox.net/runelite/launcher/"' + + - name: Invalidate CloudFront + run: | + aws cloudfront create-invalidation --distribution-id $CLOUDFRONT_DISTRIBUTION --paths "/runelite/launcher/*" + env: + CLOUDFRONT_DISTRIBUTION: ${{ secrets.CLOUDFRONT_DISTRIBUTION }} diff --git a/bootstrap-base.json b/bootstrap-base.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/bootstrap-base.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/bootstrap.py b/bootstrap.py new file mode 100644 index 00000000..9278a92d --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,106 @@ +import hashlib +import json +import os +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Tuple + +BASE_PATH = "." +BASE_REPO_URL = "https://cdn.rsprox.net/runelite/launcher" + + +def get_size_and_hash(path: str) -> Tuple[int, str]: + with open(path, "rb") as f: + sha256 = hashlib.sha256(f.read()).hexdigest() + f.seek(0, os.SEEK_END) + size = f.tell() + return size, sha256 + + +def get_launcher_version() -> str: + namespace = "{http://maven.apache.org/POM/4.0.0}" + tree = ET.parse(f"./pom.xml") + root = tree.getroot() + return root.find(f"{namespace}version").text + + +def main(): + with open("./bootstrap-base.json", 'r') as f: + bootstrap_base = json.load(f) + launcher_version = get_launcher_version() + launcher_size, launcher_sha256 = get_size_and_hash(f"{BASE_PATH}/target/launcher-{launcher_version}.jar") + + artifacts = [ + { + "name": f"launcher-{launcher_version}.jar", + "path": f"{BASE_REPO_URL}/net/runelite/launcher/{launcher_version}/launcher-{launcher_version}.jar", + "size": launcher_size, + "hash": launcher_sha256 + } + ] + dependency_hashes = { + f"launcher-{launcher_version}.jar": launcher_sha256 + } + + dependencies = open(Path(BASE_PATH) / "deps.txt", 'r').readlines() + + for line in dependencies: + line = line.strip().replace("\n", "") + if not line or "The following files have been resolved" in line: + continue + artifact_dict = {} + sections = line.split(':') + sections = sections[0:-1] # remove scope (e.g. compile, runtime, test) + + group_id = sections[0] + artifact = sections[1] + classifier = sections[3] if len(sections) == 5 else None + version = sections[-1] + + jar_name = f"{artifact}-{version}.jar" if not classifier else f"{artifact}-{version}-{classifier}.jar" + artifact_dict["name"] = jar_name + + local_jar_path = f"{BASE_PATH}/target/dependency/{jar_name}" + remote_jar_path = f"{BASE_REPO_URL}/{group_id.replace('.', '/')}/{artifact}/{version}/{jar_name}" + artifact_dict["path"] = remote_jar_path + + size, sha256 = get_size_and_hash(local_jar_path) + artifact_dict["size"] = size + artifact_dict["hash"] = sha256 + + if classifier: + platform = {} + + if "windows" in classifier: + platform["name"] = "win" + elif "macos" in classifier: + platform["name"] = "macos" + elif "linux" in classifier: + platform["name"] = "linux" + + if "arm64" in classifier: + platform["arch"] = "aarch64" + elif "x86" in classifier: + platform["arch"] = "x86" + elif "x64" in classifier: + platform["arch"] = "x86_64" + + if len(platform) > 0: + artifact_dict["platform"] = [platform] + + artifacts.append(artifact_dict) + dependency_hashes[jar_name] = sha256 + + document = { + **bootstrap_base, + "artifacts": artifacts, + "launcher": { + "mainClass": "net.runelite.launcher.Launcher", + "version": launcher_version + } + } + print(json.dumps(dict(sorted(document.items())), indent=4)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/deploy-dependencies.py b/deploy-dependencies.py new file mode 100644 index 00000000..8727ef6f --- /dev/null +++ b/deploy-dependencies.py @@ -0,0 +1,71 @@ +import asyncio +import subprocess +from pathlib import Path +from typing import List + +BASE_PATH = "./" +DEP_ROOT = "target/dependency/" +REPO_ID = "cdn.rsprox.net" +REPO_URL = f"s3://{REPO_ID}/runelite/launcher" + + +def index_dependencies(deps: List[str]) -> dict: + indexed = {} + + for line in deps: + line = line.strip().replace("\n", "") + if not line or "The following files have been resolved" in line: + continue + + sections = line.split(':') + sections = sections[0:-1] # remove scope (e.g. compile, runtime, test) + + group_id = sections[0] + artifact = sections[1] + classifier = sections[3] if len(sections) == 5 else None + version = sections[-1] + + idx = f"{group_id}:{artifact}" + if idx not in indexed: + indexed[idx] = { + "group": group_id, + "artifact": artifact, + "version": version, + "classifiers": [classifier] if classifier else [] + } + else: + indexed[idx]["classifiers"].append(classifier) + + return indexed + + +def generate_maven_command(dep: dict) -> str: + artifact = dep["artifact"] + version = dep["version"] + classifiers = dep["classifiers"] + + command = f"mvn deploy:deploy-file -Durl={REPO_URL} -DrepositoryId={REPO_ID} -Dfile={DEP_ROOT}{artifact}-{version}.jar -DgeneratePom=false -DpomFile={DEP_ROOT}{artifact}-{version}.pom" + + if len(classifiers) > 0: + command += f" -Dfiles={','.join([f'{DEP_ROOT}{artifact}-{version}-{classifier}.jar' for classifier in classifiers])}" + command += f" -Dclassifiers={','.join([c for c in classifiers])}" + command += f" -Dtypes={','.join(['jar' for i in range(len(classifiers))])}" + + return command + " -Dpackaging=jar --settings settings.xml" + + +async def run_command(command: str): + subprocess.run(command.split(' ')) + + +async def amain(): + dependencies = open(Path(BASE_PATH) / "deps.txt", 'r').readlines() + indexed = index_dependencies(dependencies) + commands = [generate_maven_command(dep) for _, dep in indexed.items()] + + to_run = [run_command(cmd) for cmd in commands] + await asyncio.gather(*to_run) + + +if __name__ == '__main__': + asyncio.run(amain()) \ No newline at end of file diff --git a/pom.xml b/pom.xml index b514398d..7a4779b7 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ cdn.rsprox.net - s3://cdn.rsprox.net/maven + s3://cdn.rsprox.net/runelite/launcher/ @@ -190,8 +190,6 @@ - RuneLite - src/main/resources @@ -249,37 +247,6 @@ - - org.apache.maven.plugins - maven-shade-plugin - 3.2.3 - - - package - - shade - - - true - - - - ${main.class} - - - - - ch.qos.logback:* - - ** - - - - - - - org.apache.maven.plugins maven-resources-plugin diff --git a/settings.xml b/settings.xml index 4b9cfcba..763bd836 100644 --- a/settings.xml +++ b/settings.xml @@ -13,7 +13,7 @@ cdn.rsprox.net - s3://cdn.rsprox.net/maven + s3://cdn.rsprox.net/runelite/launcher @@ -22,8 +22,8 @@ cdn.rsprox.net - {env.AWS_S3_ACCESS_KEY} - ${env.AWS_S3_SECRET_ACCESS_KEY} + ${env.AWS_ACCESS_KEY_ID} + ${env.AWS_SECRET_ACCESS_KEY} eu-west-1 true