Merge pull request #3 from mskcc-omics-workflows/update/actions

Update/actions
mskcc-omics-workflows · Apr 18, 2024 · 1702939 · 1702939
2 parents 3768c70 + e40056f
commit 1702939
Show file tree

Hide file tree

Showing 6 changed files with 268 additions and 30 deletions.
diff --git a/.github/jsonschema/docker_image.json b/.github/jsonschema/docker_image.json
@@ -0,0 +1,54 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema",
+    "title": "Docker",
+    "description": "Validate output of docker inspect <IMAGE>",
+    "type": "object",
+    "properties": {
+        "Architecture": {
+            "type": "string",
+            "pattern": "amd64"
+        },
+        "Os": {
+            "type": "string",
+            "pattern": "linux"
+        },
+        "Config": {
+            "type":"object",
+            "properties": {
+                "Labels": {
+                    "type":"object",
+                    "patternProperties": {
+                        "^org\\.opencontainers\\.image(?!\\.created\\.?|\\.authors\\.?|\\.url\\.?|\\.documentation\\.?|\\.source\\.?|\\.version\\.?|\\.revision\\.?|\\.vendor\\.?|\\.licenses\\.?|\\.ref\\.name\\.?|\\.title\\.?|\\.description\\.?|\\.base\\.digest\\.?|\\.base\\.name\\.?).*": { 
+                            "not": {}
+                        },
+                        "org\\.opencontainers\\.image\\.created(\\..*)?": {
+                            "type": "string",
+                            "pattern": "^(\\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])(?:T([01][0-9]|2[0-3])(:[0-5][0-9])(:[0-5][0-9])(?:\\.[0-9]+)?Z?)?$"
+                        },
+                        "org\\.opencontainers\\.image\\.authors(\\..*)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.url(\\..*)?": {
+                            "type": "string",
+                            "pattern":"^(?:http://|https://|www\\.).*"
+                        },
+                        "org\\.opencontainers\\.image\\.documentation(\\..+)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.source(\\..+)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.version(\\..+)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.revision(\\..+)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.vendor(\\..+)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.licenses(\\..+)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.ref\\.name(\\..+)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.title(\\..+)?": {"type": "string"},
+                        "org\\.opencontainers\\.image\\.description(\\..+)?":{"type": "string"},
+                        "org\\.opencontainers\\.image\\.base\\.digest(\\..+)?":{"type": "string"},
+                        "org\\.opencontainers\\.image\\.base\\.name(\\..+)?": {"type": "string"}
+                    },
+                    "additionalProperties": true              
+                }
+            },
+            "required":["Labels"],
+            "additionalProperties": true
+        }
+    },
+    "required":["Architecture","Os","Config"],
+    "additionalProperties": true
+}
diff --git a/.github/validate_docker.py b/.github/validate_docker.py
@@ -0,0 +1,52 @@
+import docker, sys
+import json, jsonschema
+import re
+
+def validate_image_labels(image_name,schema,required_annot): #, expected_labels):
+  """
+  This function checks if a Docker image has the specified structure.
+
+  Args:
+      image_name: Name of the Docker image to inspect.
+      schema: Schema to validate against.
+
+  Returns:
+      True if the image attributes match the json schema, False otherwise.
+  """
+  client = docker.from_env()
+  # if fail, raises docker.errors.ImageNotFound
+  image = client.images.get(image_name)
+  #try:
+  #  image = client.images.get(image_name)
+  #except docker.errors.ImageNotFound:
+  #  print(f"Error: Image '{image_name}' not found.")
+  #  return False
+
+  jsonschema.validators.validate(instance=image.attrs,schema=schema)
+
+  #print(json.dumps(image.attrs["Config"]["Labels"], indent=4))
+  missing_label = []
+  for i in required_annot:
+    if not any([re.search(i,j) for j in image.attrs["Config"]["Labels"]]):
+      missing_label += [i]
+
+  if len(missing_label) > 0:
+    print(f"Missing labels in image:\n{"\n".join(missing_label)}")
+    raise Exception("Failed due to missing labels")
+
+#jsonschema doesn't seem like it can easily require patterned properties
+required_annot = [
+  r"^org\.opencontainers\.image\.url(\..+)?",
+  r"^org\.opencontainers\.image\.created(\..+)?",
+  r"^org\.opencontainers\.image\.authors(\..+)?",
+  r"^org\.opencontainers\.image\.source(\..+)?",
+  r"^org\.opencontainers\.image\.version(\..+)?",
+  r"^org\.opencontainers\.image\.title(\..+)?",
+  r"^org\.opencontainers\.image\.description(\..+)?"
+]
+
+if __name__ == "__main__":
+  image_name = sys.argv[1]
+  with open(sys.argv[2]) as fh:
+    schema = json.load(fh)
+  validate_image_labels(image_name,schema,required_annot)
diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml
@@ -10,7 +10,9 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  REGISTRY: mskcc.jfrog.io/omicswf-docker-dev-local/mskcc-omics-workflows
+  JFROG_REGISTRY: mskcc.jfrog.io
+  JFROG_CONTAINER_REPO: mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows
+  GHCR_CONTAINER_REPO: ghcr.io/mskcc-omics-workflows/
 
 jobs:
   dockerfile-changes:
@@ -43,7 +45,7 @@ jobs:
         with:
           filters: ".github/tags.yml"
           token: ""
-  dockerfile-build:
+  dockerfile-validate-build:
     runs-on: ubuntu-latest
     name: dockerfile-build
     needs: [dockerfile-changes]
@@ -55,6 +57,21 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Hadolint
+        uses: hadolint/[email protected]
+        with:
+          dockerfile: containers/${{ matrix.tags }}/Dockerfile
+          verbose: true
+      - name: Set up Python
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
+        with:
+          python-version: "3.12"
+      - name: Get docker repo name
+        id: docker_repo_name
+        run: echo name=$( dirname "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
+      - name: Get docker version
+        id: docker_repo_version
+        run: echo version=$( basename "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -65,26 +82,44 @@ jobs:
           registry: mskcc.jfrog.io
           username: ${{ secrets.MSK_JFROG_USERNAME }}
           password: ${{ secrets.MSK_JFROG_TOKEN }}
-      - name: Get docker repo name
-        id: docker_repo_name
-        run: echo name=$( dirname "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
-      - name: Get docker version
-        id: docker_repo_version
-        run: echo version=$( basename "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
-      - name: Build Docker
+      - name: Build Docker Jfrog
         uses: docker/build-push-action@v5
         with:
           context: containers/${{ matrix.tags }}/
           file: containers/${{ matrix.tags }}/Dockerfile
           load: true
-          tags: ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+          tags: ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+      - name: Install pip
+        run: python -m pip install --upgrade pip
+      - name: Install python docker package
+        run : pip install docker jsonschema
+      - name: Validate docker
+        run: |
+          python .github/validate_docker.py ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}} .github/jsonschema/docker_image.json
       - name: Test image
         run: |
-          docker run --rm ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
-      - name: Export to Docker
+          docker run --rm ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+      - name: Push build to registry
         uses: docker/build-push-action@v5
         with:
           context: containers/${{ matrix.tags }}/
           file: containers/${{ matrix.tags }}/Dockerfile
-          tags: ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
-          push: true
+          tags: ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+          push: true
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [ dockerfile-changes, dockerfile-validate-build ]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
diff --git a/.github/workflows/prod-build.yml b/.github/workflows/prod-build.yml
@@ -8,7 +8,9 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  REGISTRY: mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows
+  JFROG_REGISTRY: mskcc.jfrog.io
+  JFROG_CONTAINER_REPO: mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows
+  GHCR_CONTAINER_REPO: ghcr.io/mskcc-omics-workflows/
 
 jobs:
   dockerfile-changes:
@@ -41,7 +43,7 @@ jobs:
         with:
           filters: ".github/tags.yml"
           token: ""
-  dockerfile-build:
+  dockerfile-validate-build:
     runs-on: ubuntu-latest
     name: dockerfile-build
     needs: [dockerfile-changes]
@@ -53,36 +55,77 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Hadolint
+        uses: hadolint/[email protected]
+        with:
+          dockerfile: containers/${{ matrix.tags }}/Dockerfile
+          verbose: true
+      - name: Set up Python
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
+        with:
+          python-version: "3.12"
+      - name: Get docker repo name
+        id: docker_repo_name
+        run: echo name=$( dirname "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
+      - name: Get docker version
+        id: docker_repo_version
+        run: echo version=$( basename "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
-          registry: mskcc.jfrog.io
+          registry: env.JFROG_REGISTRY
           username: ${{ secrets.MSK_JFROG_USERNAME }}
           password: ${{ secrets.MSK_JFROG_TOKEN }}
-      - name: Get docker repo name
-        id: docker_repo_name
-        run: echo name=$( dirname "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
-      - name: Get docker version
-        id: docker_repo_version
-        run: echo version=$( basename "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
-      - name: Build Docker
+      - name: Build Docker Jfrog
         uses: docker/build-push-action@v5
         with:
           context: containers/${{ matrix.tags }}/
           file: containers/${{ matrix.tags }}/Dockerfile
           load: true
-          tags: ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+          tags: ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+      - name: Install pip
+        run: python -m pip install --upgrade pip
+      - name: Install python docker package
+        run : pip install docker jsonschema
+      - name: Validate docker
+        run: |
+          python .github/validate_docker.py ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}} .github/jsonschema/docker_image.json
       - name: Test image
         run: |
-          docker run --rm ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
-      - name: Export to Docker
+          docker run --rm ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+      - name: Push build to Jfrog repo
+        uses: docker/build-push-action@v5
+        with:
+          context: containers/${{ matrix.tags }}/
+          file: containers/${{ matrix.tags }}/Dockerfile
+          tags: ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+          push: true
+      - name: Build and push Docker to GHCR
         uses: docker/build-push-action@v5
         with:
           context: containers/${{ matrix.tags }}/
           file: containers/${{ matrix.tags }}/Dockerfile
-          tags: ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
-          push: true
+          load: true
+          tags: ${{ env.GHCR_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
+          push: true
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [ dockerfile-changes, dockerfile-validate-build ]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ success() || !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
+        run: |
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
diff --git a/README.md b/README.md
@@ -2,3 +2,43 @@
 Mono-repository for custom docker containers used in nextflow modules
 
 For further information on how to contribute or use images in this repository, visit our [Gitbook documentation](https://mskcc-omics-workflows.gitbook.io/omics-wf/GMaCKqX0TmAhUOoZmuc6/image-management)
+
+## Naming conventions
+
+Each Dockerfile should be saved within a specific folder structure:
+
+```
+containers/<softwarename>/<version>/Dockerfile
+```
+
+Any custom scripts or resource files that are included in the image or required for building the image should be contained in the same folder as the Dockerfile. When you open a pull request for the container, an image will be automatically created and be labeled `$REGISTRY/<softwarename>:version`
+
+The version of the image should correspond to the version of the software it contains. If the image has multiple independent software packages, the image version should start at 0.1.0 and increment in accordance with [Semantic Versioning](https://semver.org/#semantic-versioning-200).
+
+## Image repository
+
+Development images are stored at `mskcc.jfrog.io/omicswf-docker-dev-local/mskcc-omics-workflows`. When a pull request is created against the `develop` branch, and the image passes basic checks, the image is automatically built and pushed to this location.
+
+Productions images are stored at `mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows`. When a pull request is created against the `master` branch, and the image passes basic checks, the image is automatically built and pushed to this location.
+
+When contributing an image you should first open a pull request to `develop`. The `develop` branch will be merged regularly into `master`.
+
+## Image Requirements
+
+Requirements are based on [OCI image-spec annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md). The image must have the following labels:
+```
+org.opencontainers.image.url
+org.opencontainers.image.created
+org.opencontainers.image.authors
+org.opencontainers.image.source
+org.opencontainers.image.version
+org.opencontainers.image.title
+org.opencontainers.image.description
+```
+You can optionally use any of the above properties with an extension related to a specific software in the image, and you can use multiple of them as well. For example, if you want to label versions for both the Java version and the abra version:
+```
+LABEL \
+    org.opencontainers.image.version.java=${JAVA_VERSION} \
+    org.opencontainers.image.version.abra2=${ABRA2_VERSION}
+```
+It is also acceptable to use no extension.
diff --git a/containers/testapp/0.0.1/Dockerfile b/containers/testapp/0.0.1/Dockerfile
@@ -1,5 +1,19 @@
 FROM python:3.9-slim
 
+# Labels
+LABEL org.opencontainers.image.vendor="MSKCC-OMICS-WORKFLOWS"
+LABEL org.opencontainers.image.authors="Anne Marie Noronha ([email protected])" 
+
+LABEL org.opencontainers.image.created="2020-12-16T15:55:35Z" \ 
+	org.opencontainers.image.version="0.0.1" \
+	org.opencontainers.image.licenses="test-license" \
+	org.opencontainers.image.source="https://github.com/mskcc-omics-workflows/containers/containers/testapp/" \
+    org.opencontainers.image.url="https://github.com/mskcc-omics-workflows/containers/" \
+    org.opencontainers.image.title="My test app" \
+    org.opencontainers.image.description="My test app description"
+
+
+
 # Set the working directory in the container
 WORKDIR /app
 
@@ -14,4 +28,4 @@ COPY . /app
 ENV NAME World
 
 # Run app.py when the container launches
-CMD ["python", "app.py"]
+CMD ["python", "app.py"]