Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update/actions #3

Merged
merged 19 commits into from
Apr 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions .github/jsonschema/docker_image.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"title": "Docker",
"description": "Validate output of docker inspect <IMAGE>",
"type": "object",
"properties": {
"Architecture": {
"type": "string",
"pattern": "amd64"
},
"Os": {
"type": "string",
"pattern": "linux"
},
"Config": {
"type":"object",
"properties": {
"Labels": {
"type":"object",
"patternProperties": {
"^org\\.opencontainers\\.image(?!\\.created\\.?|\\.authors\\.?|\\.url\\.?|\\.documentation\\.?|\\.source\\.?|\\.version\\.?|\\.revision\\.?|\\.vendor\\.?|\\.licenses\\.?|\\.ref\\.name\\.?|\\.title\\.?|\\.description\\.?|\\.base\\.digest\\.?|\\.base\\.name\\.?).*": {
"not": {}
},
"org\\.opencontainers\\.image\\.created(\\..*)?": {
"type": "string",
"pattern": "^(\\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])(?:T([01][0-9]|2[0-3])(:[0-5][0-9])(:[0-5][0-9])(?:\\.[0-9]+)?Z?)?$"
},
"org\\.opencontainers\\.image\\.authors(\\..*)?": {"type": "string"},
"org\\.opencontainers\\.image\\.url(\\..*)?": {
"type": "string",
"pattern":"^(?:http://|https://|www\\.).*"
},
"org\\.opencontainers\\.image\\.documentation(\\..+)?": {"type": "string"},
"org\\.opencontainers\\.image\\.source(\\..+)?": {"type": "string"},
"org\\.opencontainers\\.image\\.version(\\..+)?": {"type": "string"},
"org\\.opencontainers\\.image\\.revision(\\..+)?": {"type": "string"},
"org\\.opencontainers\\.image\\.vendor(\\..+)?": {"type": "string"},
"org\\.opencontainers\\.image\\.licenses(\\..+)?": {"type": "string"},
"org\\.opencontainers\\.image\\.ref\\.name(\\..+)?": {"type": "string"},
"org\\.opencontainers\\.image\\.title(\\..+)?": {"type": "string"},
"org\\.opencontainers\\.image\\.description(\\..+)?":{"type": "string"},
"org\\.opencontainers\\.image\\.base\\.digest(\\..+)?":{"type": "string"},
"org\\.opencontainers\\.image\\.base\\.name(\\..+)?": {"type": "string"}
},
"additionalProperties": true
}
},
"required":["Labels"],
"additionalProperties": true
}
},
"required":["Architecture","Os","Config"],
"additionalProperties": true
}
52 changes: 52 additions & 0 deletions .github/validate_docker.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import docker, sys
import json, jsonschema
import re

def validate_image_labels(image_name,schema,required_annot): #, expected_labels):
"""
This function checks if a Docker image has the specified structure.

Args:
image_name: Name of the Docker image to inspect.
schema: Schema to validate against.

Returns:
True if the image attributes match the json schema, False otherwise.
"""
client = docker.from_env()
# if fail, raises docker.errors.ImageNotFound
image = client.images.get(image_name)
#try:
# image = client.images.get(image_name)
#except docker.errors.ImageNotFound:
# print(f"Error: Image '{image_name}' not found.")
# return False

jsonschema.validators.validate(instance=image.attrs,schema=schema)

#print(json.dumps(image.attrs["Config"]["Labels"], indent=4))
missing_label = []
for i in required_annot:
if not any([re.search(i,j) for j in image.attrs["Config"]["Labels"]]):
missing_label += [i]

if len(missing_label) > 0:
print(f"Missing labels in image:\n{"\n".join(missing_label)}")
raise Exception("Failed due to missing labels")

#jsonschema doesn't seem like it can easily require patterned properties
required_annot = [
r"^org\.opencontainers\.image\.url(\..+)?",
r"^org\.opencontainers\.image\.created(\..+)?",
r"^org\.opencontainers\.image\.authors(\..+)?",
r"^org\.opencontainers\.image\.source(\..+)?",
r"^org\.opencontainers\.image\.version(\..+)?",
r"^org\.opencontainers\.image\.title(\..+)?",
r"^org\.opencontainers\.image\.description(\..+)?"
]

if __name__ == "__main__":
image_name = sys.argv[1]
with open(sys.argv[2]) as fh:
schema = json.load(fh)
validate_image_labels(image_name,schema,required_annot)
63 changes: 49 additions & 14 deletions .github/workflows/dev-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ concurrency:
cancel-in-progress: true

env:
REGISTRY: mskcc.jfrog.io/omicswf-docker-dev-local/mskcc-omics-workflows
JFROG_REGISTRY: mskcc.jfrog.io
JFROG_CONTAINER_REPO: mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows
GHCR_CONTAINER_REPO: ghcr.io/mskcc-omics-workflows/

jobs:
dockerfile-changes:
Expand Down Expand Up @@ -43,7 +45,7 @@ jobs:
with:
filters: ".github/tags.yml"
token: ""
dockerfile-build:
dockerfile-validate-build:
runs-on: ubuntu-latest
name: dockerfile-build
needs: [dockerfile-changes]
Expand All @@ -55,6 +57,21 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Hadolint
uses: hadolint/[email protected]
with:
dockerfile: containers/${{ matrix.tags }}/Dockerfile
verbose: true
- name: Set up Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
with:
python-version: "3.12"
- name: Get docker repo name
id: docker_repo_name
run: echo name=$( dirname "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
- name: Get docker version
id: docker_repo_version
run: echo version=$( basename "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
Expand All @@ -65,26 +82,44 @@ jobs:
registry: mskcc.jfrog.io
username: ${{ secrets.MSK_JFROG_USERNAME }}
password: ${{ secrets.MSK_JFROG_TOKEN }}
- name: Get docker repo name
id: docker_repo_name
run: echo name=$( dirname "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
- name: Get docker version
id: docker_repo_version
run: echo version=$( basename "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
- name: Build Docker
- name: Build Docker Jfrog
uses: docker/build-push-action@v5
with:
context: containers/${{ matrix.tags }}/
file: containers/${{ matrix.tags }}/Dockerfile
load: true
tags: ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
tags: ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
- name: Install pip
run: python -m pip install --upgrade pip
- name: Install python docker package
run : pip install docker jsonschema
- name: Validate docker
run: |
python .github/validate_docker.py ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}} .github/jsonschema/docker_image.json
- name: Test image
run: |
docker run --rm ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
- name: Export to Docker
docker run --rm ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
- name: Push build to registry
uses: docker/build-push-action@v5
with:
context: containers/${{ matrix.tags }}/
file: containers/${{ matrix.tags }}/Dockerfile
tags: ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
push: true
tags: ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
push: true
confirm-pass:
runs-on: ubuntu-latest
needs: [ dockerfile-changes, dockerfile-validate-build ]
if: always()
steps:
- name: All tests ok
if: ${{ success() || !contains(needs.*.result, 'failure') }}
run: exit 0
- name: One or more tests failed
if: ${{ contains(needs.*.result, 'failure') }}
run: exit 1

- name: debug-print
if: always()
run: |
echo "toJSON(needs) = ${{ toJSON(needs) }}"
echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
73 changes: 58 additions & 15 deletions .github/workflows/prod-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ concurrency:
cancel-in-progress: true

env:
REGISTRY: mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows
JFROG_REGISTRY: mskcc.jfrog.io
JFROG_CONTAINER_REPO: mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows
GHCR_CONTAINER_REPO: ghcr.io/mskcc-omics-workflows/

jobs:
dockerfile-changes:
Expand Down Expand Up @@ -41,7 +43,7 @@ jobs:
with:
filters: ".github/tags.yml"
token: ""
dockerfile-build:
dockerfile-validate-build:
runs-on: ubuntu-latest
name: dockerfile-build
needs: [dockerfile-changes]
Expand All @@ -53,36 +55,77 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Hadolint
uses: hadolint/[email protected]
with:
dockerfile: containers/${{ matrix.tags }}/Dockerfile
verbose: true
- name: Set up Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
with:
python-version: "3.12"
- name: Get docker repo name
id: docker_repo_name
run: echo name=$( dirname "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
- name: Get docker version
id: docker_repo_version
run: echo version=$( basename "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
registry: mskcc.jfrog.io
registry: env.JFROG_REGISTRY
username: ${{ secrets.MSK_JFROG_USERNAME }}
password: ${{ secrets.MSK_JFROG_TOKEN }}
- name: Get docker repo name
id: docker_repo_name
run: echo name=$( dirname "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
- name: Get docker version
id: docker_repo_version
run: echo version=$( basename "${{ matrix.tags }}" ) >> $GITHUB_OUTPUT
- name: Build Docker
- name: Build Docker Jfrog
uses: docker/build-push-action@v5
with:
context: containers/${{ matrix.tags }}/
file: containers/${{ matrix.tags }}/Dockerfile
load: true
tags: ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
tags: ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
- name: Install pip
run: python -m pip install --upgrade pip
- name: Install python docker package
run : pip install docker jsonschema
- name: Validate docker
run: |
python .github/validate_docker.py ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}} .github/jsonschema/docker_image.json
- name: Test image
run: |
docker run --rm ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
- name: Export to Docker
docker run --rm ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
- name: Push build to Jfrog repo
uses: docker/build-push-action@v5
with:
context: containers/${{ matrix.tags }}/
file: containers/${{ matrix.tags }}/Dockerfile
tags: ${{ env.JFROG_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
push: true
- name: Build and push Docker to GHCR
uses: docker/build-push-action@v5
with:
context: containers/${{ matrix.tags }}/
file: containers/${{ matrix.tags }}/Dockerfile
tags: ${{ env.REGISTRY }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
push: true
load: true
tags: ${{ env.GHCR_CONTAINER_REPO }}/${{ steps.docker_repo_name.outputs.name}}:${{steps.docker_repo_version.outputs.version}}
push: true
confirm-pass:
runs-on: ubuntu-latest
needs: [ dockerfile-changes, dockerfile-validate-build ]
if: always()
steps:
- name: All tests ok
if: ${{ success() || !contains(needs.*.result, 'failure') }}
run: exit 0
- name: One or more tests failed
if: ${{ contains(needs.*.result, 'failure') }}
run: exit 1

- name: debug-print
if: always()
run: |
echo "toJSON(needs) = ${{ toJSON(needs) }}"
echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
40 changes: 40 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,43 @@
Mono-repository for custom docker containers used in nextflow modules

For further information on how to contribute or use images in this repository, visit our [Gitbook documentation](https://mskcc-omics-workflows.gitbook.io/omics-wf/GMaCKqX0TmAhUOoZmuc6/image-management)

## Naming conventions

Each Dockerfile should be saved within a specific folder structure:

```
containers/<softwarename>/<version>/Dockerfile
```

Any custom scripts or resource files that are included in the image or required for building the image should be contained in the same folder as the Dockerfile. When you open a pull request for the container, an image will be automatically created and be labeled `$REGISTRY/<softwarename>:version`

The version of the image should correspond to the version of the software it contains. If the image has multiple independent software packages, the image version should start at 0.1.0 and increment in accordance with [Semantic Versioning](https://semver.org/#semantic-versioning-200).

## Image repository

Development images are stored at `mskcc.jfrog.io/omicswf-docker-dev-local/mskcc-omics-workflows`. When a pull request is created against the `develop` branch, and the image passes basic checks, the image is automatically built and pushed to this location.

Productions images are stored at `mskcc.jfrog.io/omicswf-docker-prod-local/mskcc-omics-workflows`. When a pull request is created against the `master` branch, and the image passes basic checks, the image is automatically built and pushed to this location.

When contributing an image you should first open a pull request to `develop`. The `develop` branch will be merged regularly into `master`.

## Image Requirements

Requirements are based on [OCI image-spec annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md). The image must have the following labels:
```
org.opencontainers.image.url
org.opencontainers.image.created
org.opencontainers.image.authors
org.opencontainers.image.source
org.opencontainers.image.version
org.opencontainers.image.title
org.opencontainers.image.description
```
You can optionally use any of the above properties with an extension related to a specific software in the image, and you can use multiple of them as well. For example, if you want to label versions for both the Java version and the abra version:
```
LABEL \
org.opencontainers.image.version.java=${JAVA_VERSION} \
org.opencontainers.image.version.abra2=${ABRA2_VERSION}
```
It is also acceptable to use no extension.
16 changes: 15 additions & 1 deletion containers/testapp/0.0.1/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,19 @@
FROM python:3.9-slim

# Labels
LABEL org.opencontainers.image.vendor="MSKCC-OMICS-WORKFLOWS"
LABEL org.opencontainers.image.authors="Anne Marie Noronha ([email protected])"

LABEL org.opencontainers.image.created="2020-12-16T15:55:35Z" \
org.opencontainers.image.version="0.0.1" \
org.opencontainers.image.licenses="test-license" \
org.opencontainers.image.source="https://github.com/mskcc-omics-workflows/containers/containers/testapp/" \
org.opencontainers.image.url="https://github.com/mskcc-omics-workflows/containers/" \
org.opencontainers.image.title="My test app" \
org.opencontainers.image.description="My test app description"



# Set the working directory in the container
WORKDIR /app

Expand All @@ -14,4 +28,4 @@ COPY . /app
ENV NAME World

# Run app.py when the container launches
CMD ["python", "app.py"]
CMD ["python", "app.py"]
Loading