diff --git a/.dockerignore b/.dockerignore index 28ea54bea..f8451b27e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ **/node_modules -build + #* #!build/*-runner #!build/*-runner.jar diff --git a/.github/workflows/actions-deploy.yml b/.github/workflows/actions-deploy.yml index c088535d5..7c1896030 100644 --- a/.github/workflows/actions-deploy.yml +++ b/.github/workflows/actions-deploy.yml @@ -1,4 +1,5 @@ name: CD + on: push: branches: @@ -23,8 +24,11 @@ jobs: with: fetch-depth: '0' + - name: Configure Qemu + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Deploy - run: docker-compose -f docker-compose-deploy.yml up --force-recreate --exit-code-from deploy deploy + run: docker-compose -f docker-compose-deploy.yml up --build --force-recreate --exit-code-from deploy deploy env: # github token to deploy the binary REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/DRAFT.md b/DRAFT.md index 5644f96b8..9ef7bcb6a 100644 --- a/DRAFT.md +++ b/DRAFT.md @@ -72,3 +72,28 @@ java -cp './build/dns-proxy-server-3.0.0-alpha-runner.jar:../annotation-process docker-compose -f docker-compose-tmp.yml up --build + +## To build native image manually when using quarkus +Command copied from `./gradlew clean build -Dquarkus.package.type=native -i -x check` + +```bash +$ cd build/dns-proxy-server-native-image-source-jar +$ native-image -J-Djava.util.logging.manager=org.jboss.logmanager.LogManager -J-Dsun.nio.ch.maxUpdateArraySize=100 \ +-J-Dlogging.initial-configurator.min-level=500 -J-Dio.netty.leakDetection.level=DISABLED -J-Dio.netty.allocator.maxOrder=3 \ +-J-Dvertx.logger-delegate-factory-class-name=io.quarkus.vertx.core.runtime.VertxLogDelegateFactory \ +-J-Dvertx.disableDnsResolver=true -J-Duser.language=en -J-Duser.country=US -J-Dfile.encoding=UTF-8 \ +--features=io.quarkus.runner.Feature,io.quarkus.runtime.graal.ResourcesFeature,io.quarkus.runtime.graal.DisableLoggingFeature \ +-J--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED -J--add-opens=java.base/java.text=ALL-UNNAMED \ +-J--add-opens=java.base/java.io=ALL-UNNAMED -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ +-J--add-opens=java.base/java.util=ALL-UNNAMED -H:+CollectImageBuildStatistics \ +-H:ImageBuildStatisticsFile=dns-proxy-server-timing-stats.json \ +-H:BuildOutputJSONFile=dns-proxy-server-build-output-stats.json -H:+AllowFoldMethods -J-Djava.awt.headless=true \ +--no-fallback --link-at-build-time -H:+ReportExceptionStackTraces -H:-AddAllCharsets --enable-url-protocols=http \ +-H:NativeLinkerOption=-no-pie -H:-UseServiceLoaderFeature -H:+StackTrace \ +-J--add-exports=org.graalvm.sdk/org.graalvm.nativeimage.impl=ALL-UNNAMED \ +-J--add-exports=org.graalvm.nativeimage.builder/com.oracle.svm.core.jdk=ALL-UNNAMED \ +--exclude-config io\.netty\.netty-codec /META-INF/native-image/io\.netty/netty-codec/generated/handlers/reflect-config\.json \ +--exclude-config io\.netty\.netty-handler /META-INF/native-image/io\.netty/netty-handler/generated/handlers/reflect-config\.json dns-proxy-server \ +-jar dns-proxy-server.jar + +``` diff --git a/Dockerfile b/Dockerfile index 827ff1678..eb33c0dcf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,5 @@ -FROM docker.io/defreitas/tools_graalvm-22.3_java-19_debian-9:0.1.1 AS BUILDER -COPY ./ /app -WORKDIR /app -RUN ./gradlew clean build -Dquarkus.package.type=native -i &&\ - cd build &&\ - ls -lha &&\ - mkdir -p artifacts &&\ - mv $(ls -p ./ | grep -v / | grep dns-proxy-server) ./artifacts/ - FROM debian:10-slim -COPY --from=BUILDER /app/build/artifacts/* /app/dns-proxy-server +COPY ./build/artifacts/linux-amd64/dns-proxy-server /app/dns-proxy-server WORKDIR /app LABEL dps.container=true VOLUME ["/var/run/docker.sock", "/var/run/docker.sock"] diff --git a/Dockerfile.arm8x64 b/Dockerfile.aarch64 similarity index 67% rename from Dockerfile.arm8x64 rename to Dockerfile.aarch64 index ba7862600..d772a4e3e 100644 --- a/Dockerfile.arm8x64 +++ b/Dockerfile.aarch64 @@ -1,5 +1,5 @@ FROM arm64v8/debian:10-slim -ADD build/dns-proxy-server-linux-arm64-*.tgz /app/ +COPY ./build/artifacts/linux-aarch64/dns-proxy-server /app/dns-proxy-server WORKDIR /app LABEL dps.container=true VOLUME ["/var/run/docker.sock", "/var/run/docker.sock"] diff --git a/Dockerfile.arm7x86 b/Dockerfile.arm7x86 deleted file mode 100644 index d75851148..000000000 --- a/Dockerfile.arm7x86 +++ /dev/null @@ -1,6 +0,0 @@ -FROM arm32v7/debian:10-slim -ADD build/dns-proxy-server-linux-arm-*.tgz /app/ -WORKDIR /app -LABEL dps.container=true -VOLUME ["/var/run/docker.sock", "/var/run/docker.sock"] -ENTRYPOINT ["/app/dns-proxy-server"] diff --git a/Dockerfile.arm7x86.build b/Dockerfile.arm7x86.build deleted file mode 100644 index 5f49de864..000000000 --- a/Dockerfile.arm7x86.build +++ /dev/null @@ -1,5 +0,0 @@ -FROM arm32v7/debian:10-slim -COPY ./ /building -WORKDIR /building -RUN ./gradlew build nativeImage -ENTRYPOINT "/bin/sh" diff --git a/Dockerfile.builder.linux-aarch64 b/Dockerfile.builder.linux-aarch64 new file mode 100644 index 000000000..c19765488 --- /dev/null +++ b/Dockerfile.builder.linux-aarch64 @@ -0,0 +1,29 @@ +FROM defreitas/tools_graalvm-22.3_java-17_debian-9_aarch64:0.1.3 +WORKDIR /app/build +COPY ./build/artifacts/native-image-source/ /app/build + +ENV JAVA_TOOL_OPTIONS='-Djdk.lang.Process.launchMechanism=fork' +RUN uname -m && ls -lha &&\ + native-image -Djdk.lang.Process.launchMechanism=fork -J-Djdk.lang.Process.launchMechanism=fork -J-Djava.util.logging.manager=org.jboss.logmanager.LogManager -J-Dsun.nio.ch.maxUpdateArraySize=100 \ + -J-Dlogging.initial-configurator.min-level=500 -J-Dio.netty.leakDetection.level=DISABLED -J-Dio.netty.allocator.maxOrder=3 \ + -J-Dvertx.logger-delegate-factory-class-name=io.quarkus.vertx.core.runtime.VertxLogDelegateFactory \ + -J-Dvertx.disableDnsResolver=true -J-Duser.language=en -J-Duser.country=US -J-Dfile.encoding=UTF-8 \ + --features=io.quarkus.runner.Feature,io.quarkus.runtime.graal.ResourcesFeature,io.quarkus.runtime.graal.DisableLoggingFeature \ + -J--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED -J--add-opens=java.base/java.text=ALL-UNNAMED \ + -J--add-opens=java.base/java.io=ALL-UNNAMED -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ + -J--add-opens=java.base/java.util=ALL-UNNAMED -H:+CollectImageBuildStatistics \ + -H:ImageBuildStatisticsFile=dns-proxy-server-timing-stats.json \ + -H:BuildOutputJSONFile=dns-proxy-server-build-output-stats.json -H:+AllowFoldMethods -J-Djava.awt.headless=true \ + --no-fallback --link-at-build-time -H:+ReportExceptionStackTraces -H:-AddAllCharsets --enable-url-protocols=http \ + -H:NativeLinkerOption=-no-pie -H:-UseServiceLoaderFeature -H:+StackTrace \ + -J--add-exports=org.graalvm.sdk/org.graalvm.nativeimage.impl=ALL-UNNAMED \ + -J--add-exports=org.graalvm.nativeimage.builder/com.oracle.svm.core.jdk=ALL-UNNAMED \ + --exclude-config io\.netty\.netty-codec /META-INF/native-image/io\.netty/netty-codec/generated/handlers/reflect-config\.json \ + --exclude-config io\.netty\.netty-handler /META-INF/native-image/io\.netty/netty-handler/generated/handlers/reflect-config\.json dns-proxy-server \ + -jar dns-proxy-server.jar + +RUN ls -lha &&\ + mkdir -p ./artifacts/linux-aarch64 &&\ + mv ./dns-proxy-server ./artifacts/linux-aarch64/ + +ENTRYPOINT cat diff --git a/Dockerfile.builder.linux-amd64 b/Dockerfile.builder.linux-amd64 new file mode 100644 index 000000000..0419e40aa --- /dev/null +++ b/Dockerfile.builder.linux-amd64 @@ -0,0 +1,10 @@ +FROM docker.io/defreitas/tools_graalvm-22.3_java-19_debian-9:0.1.1 AS BUILDER +COPY ./ /app +WORKDIR /app +RUN ./gradlew clean build -Dquarkus.package.type=native -i &&\ + ./gradlew build -Dquarkus.package.type=uber-jar -i -x check &&\ + ls -lha ./build &&\ + mkdir -p ./build/artifacts/linux-amd64 && mv ./build/dns-proxy-server ./build/artifacts/linux-amd64/ &&\ + mkdir -p ./build/artifacts/jre && mv ./build/dns-proxy-server.jar ./build/artifacts/jre/ &&\ + mv ./build/dns-proxy-server-native-image-source-jar ./build/artifacts/native-image-source +ENTRYPOINT cat diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 2ef98192f..7bf5e2db2 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,8 @@ +### 3.0.4-beta +* Generating new distributions binaries + * aarch64 aside docker image + * uber jar, target jvm = 17 for now + ### 3.0.2-beta * Fixed home page was pointing to wrong place diff --git a/builder.bash b/builder.bash index c265fbb4d..41bf85215 100755 --- a/builder.bash +++ b/builder.bash @@ -4,6 +4,7 @@ set -e REPO_DIR=`pwd` APP_VERSION=$(cat gradle.properties | grep -oP 'version=\K(.+)') +export VERSION=${APP_VERSION} echo "> builder.bash version=${APP_VERSION}, path=${REPO_DIR}" @@ -24,63 +25,108 @@ copyFileFromService(){ from=$2 to=$3 - docker-compose up --no-start --build --force-recreate $serviceName 1>&2 - id=$(docker ps -a | grep $serviceName | awk '{print $1}') + id=$(docker-compose up --no-start --build --force-recreate $serviceName 2>&1 | grep Container | awk '{print $2}' | head -1) + echo "> copy from service=${serviceName}, id=${id}, from=${from}, to=${to}" docker cp "$id:$from" "$to" } +validateRelease(){ + echo "> validate release, version=${APP_VERSION}, git=$(git rev-parse $APP_VERSION 2>/dev/null)" + if git rev-parse "$APP_VERSION^{}" >/dev/null 2>&1; then + echo "> Tag already exists $APP_VERSION" + exit 3 + fi +} + case $1 in + copy ) + copyFileFromService build-frontend /static ./src/main/resources/META-INF/resources/static + ;; + validate-release ) - echo "> validate release, version=${APP_VERSION}, git=$(git rev-parse $APP_VERSION 2>/dev/null)" - if git rev-parse "$APP_VERSION^{}" >/dev/null 2>&1; then - echo "> Tag already exists $APP_VERSION" - exit 3 - fi + validateRelease + ;; + + build-frontend ) + + echo "> Building frontend files..." + docker-compose build --no-cache --progress=plain build-frontend + rm -rf ./src/main/resources/META-INF/resources/static + copyFileFromService build-frontend /static ./src/main/resources/META-INF/resources/static + + echo "> Build, test and generate the binaries" + mkdir -p "${REPO_DIR}/build" + + ;; + + build-backend ) + + OS=linux + ARCH=$2 + BUILD_SERVICE_NAME="build-${OS}-${ARCH}" + IMAGE_SERVICE_NAME="image-${OS}-${ARCH}" + ARTIFACTS_DIR="${REPO_DIR}/build/artifacts" + + echo "> building backend to: os=${OS}, arch=${ARCH}" + + mkdir -p ${ARTIFACTS_DIR} + + docker-compose build --no-cache --progress=plain ${BUILD_SERVICE_NAME} + + copyFileFromService ${BUILD_SERVICE_NAME} /app/build/artifacts /tmp/ + mv -v /tmp/artifacts/* ${ARTIFACTS_DIR} + + docker-compose build --no-cache --progress=plain "${IMAGE_SERVICE_NAME}" + + ;; + + compress-upload-artifacts ) + echo "> compress the files ..." + ./builder.bash validate-release || exit 0 + + ARTIFACTS_DIR="${REPO_DIR}/build/artifacts" + COMPRESSED_ARTIFACTS_DIR="${REPO_DIR}/build/compressed-artifacts" + + mkdir -p ${COMPRESSED_ARTIFACTS_DIR} + cd ${ARTIFACTS_DIR} + + ls ${ARTIFACTS_DIR} | grep -v "native-image-source" |\ + while read -r artPath ; do + tgz="${COMPRESSED_ARTIFACTS_DIR}/dns-proxy-server-${artPath}-${APP_VERSION}.tgz" + tar -czvf ${tgz} -C ${artPath} . + echo "> compressed ${artPath} to ${tgz} ..." + done + + echo "> Uploading the release artifacts" + cd $REPO_DIR + DESC=$(cat RELEASE-NOTES.md | awk 'BEGIN {RS="|"} {print substr($0, 0, index(substr($0, 3), "###"))}' | sed ':a;N;$!ba;s/\n/\\r\\n/g') + github-cli release mageddo dns-proxy-server $APP_VERSION $CURRENT_BRANCH "${DESC}" ${COMPRESSED_ARTIFACTS_DIR}/*.tgz + + ;; + + docker-push ) + echo "> Push docker images to docker hub" +# docker tag defreitas/dns-proxy-server:${APP_VERSION} defreitas/dns-proxy-server:latest &&\ + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin &&\ + docker-compose push image-linux-amd64 image-linux-aarch64 ;; deploy ) echo "> Deploy started , current branch=$CURRENT_BRANCH" + rm -vrf build + ls -lha + ./builder.bash validate-release || exit 0 - if [ "$CURRENT_BRANCH" != "master" ]; then - echo "> refusing to go ahead outside the master branch" - exit 8 - fi + ./builder.bash build-frontend + ./builder.bash build-backend amd64 #also builds the jar + ./builder.bash build-backend aarch64 - echo "> Building frontend files..." - docker-compose build --progress=plain build-frontend - rm -rf ./src/main/resources/META-INF/resources/static - copyFileFromService build-frontend /static ./src/main/resources/META-INF/resources/static - - echo "> Build, test and generate the binaries" - mkdir -p "${REPO_DIR}/build" - - OS=linux - ARCH=amd64 - SERVICE_NAME="build-${OS}-${ARCH}" - BIN_FILE="${REPO_DIR}/build/dns-proxy-server-${OS}-${ARCH}-${APP_VERSION}" - TAR_FILE=${BIN_FILE}.tgz - - VERSION=${APP_VERSION} docker-compose build --progress=plain ${SERVICE_NAME} - copyFileFromService ${SERVICE_NAME} /app/dns-proxy-server ${BIN_FILE} - cd $REPO_DIR/build/ - tar --exclude=*.tgz -czf $TAR_FILE $(basename ${BIN_FILE}) - - echo "> Uploading the release artifacts" - cd $REPO_DIR - DESC=$(cat RELEASE-NOTES.md | awk 'BEGIN {RS="|"} {print substr($0, 0, index(substr($0, 3), "###"))}' | sed ':a;N;$!ba;s/\n/\\r\\n/g') - github-cli release mageddo dns-proxy-server $APP_VERSION $CURRENT_BRANCH "${DESC}" $REPO_DIR/build/*.tgz - - echo "> Push docker images to docker hub" -# docker-compose build prod-build-image-dps prod-build-image-dps-arm7x86 prod-build-image-dps-arm8x64 &&\ -# docker tag defreitas/dns-proxy-server:${APP_VERSION} defreitas/dns-proxy-server:latest &&\ - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin &&\ - VERSION=${APP_VERSION} docker-compose push build-linux-amd64 -# docker-compose push prod-build-image-dps prod-build-image-dps-arm7x86 prod-build-image-dps-arm8x64 && -# docker push defreitas/dns-proxy-server:latest + ./builder.bash compress-upload-artifacts + ./builder.bash docker-push ;; deploy-docs ) diff --git a/docker-compose-alpha.yml b/docker-compose-alpha.yml deleted file mode 100644 index b22d101f2..000000000 --- a/docker-compose-alpha.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: '3.3' -services: - arm7x86: - platform: linux/arm32v7 - build: - context: . - dockerfile: Dockerfile.arm7x86.build diff --git a/docker-compose.yml b/docker-compose.yml index 483895a34..714bfc485 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2.3' +version: '3' services: # tag:build @@ -9,25 +9,30 @@ services: image: defreitas/dns-proxy-server-build-frontend:${VERSION:-snapshot} build-linux-amd64: - image: defreitas/dns-proxy-server:${VERSION:-snapshot} + image: defreitas/dns-proxy-server-build:${VERSION:-snapshot}-amd64 build: context: . - dockerfile: Dockerfile + dockerfile: Dockerfile.builder.linux-amd64 - # end:build - - # tag:development + build-linux-aarch64: + image: defreitas/dns-proxy-server-build:${VERSION:-snapshot}-aarch64 + build: + context: . + dockerfile: Dockerfile.builder.linux-aarch64 + platform: linux/arm64/v8 - # end:development + # end:build - prod-build-image-dps-arm7x86: + # tag:image + image-linux-amd64: + image: defreitas/dns-proxy-server:${VERSION:-snapshot} build: context: . - dockerfile: Dockerfile.arm7x86 - image: defreitas/dns-proxy-server:${VERSION:-snapshot}-arm7x86 + dockerfile: Dockerfile - prod-build-image-dps-arm8x64: + image-linux-aarch64: + image: defreitas/dns-proxy-server:${VERSION:-snapshot}-aarch64 build: context: . - dockerfile: Dockerfile.arm8x64 - image: defreitas/dns-proxy-server:${VERSION:-snapshot}-arm8x64 + dockerfile: Dockerfile.aarch64 + # end:image diff --git a/gradle.properties b/gradle.properties index b5f17aa5f..bb2324270 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=3.0.2-beta +version=3.0.4-beta quarkusPluginId=io.quarkus quarkusPluginVersion=2.16.0.Final quarkusPlatformGroupId=io.quarkus.platform diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index ac4e08a6d..1cab91f62 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -13,5 +13,6 @@ quarkus.log.category."com.mageddo".level=DEBUG # quarkus-core-deployment-2.16.0.Final.jar!/META-INF/quarkus-javadoc.properties #quarkus.package.runner-suffix= +quarkus.package.output-name=dns-proxy-server quarkus.package.add-runner-suffix=false #quarkus.package.output-directory=artifacts