Skip to content

Commit

Permalink
Initial commit for the integration-test.yml workflow
Browse files Browse the repository at this point in the history
Signed-off-by: Radoslav Dimitrov <[email protected]>
  • Loading branch information
rdimitrov committed Jan 17, 2025
1 parent e33a35e commit 22cc684
Show file tree
Hide file tree
Showing 12 changed files with 340 additions and 22 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# This workflow builds, tests and lints the project
name: CI
name: Build, Test & Lint

on:
workflow_call:
workflow_dispatch:

jobs:
test:
name: Run
runs-on: ubuntu-latest
strategy:
matrix:
Expand Down
36 changes: 35 additions & 1 deletion .github/workflows/image-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@ name: OCI Image - Build

on:
workflow_call:
workflow_dispatch:
inputs:
artifact-name:
description: 'The name of the artifact to upload'
required: true
type: string

permissions:
contents: read
Expand All @@ -12,15 +16,33 @@ jobs:
docker-image:
name: Check docker image build
runs-on: ubuntu-latest
env:
IMAGE_NAME: stacklok/codegate
IMAGE_TAG: dev
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3
- name: Download artifact
id: download-artifact
uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
with:
github_token: ${{ github.token }}
workflow: ".github/workflows/import_packages.yml"
workflow_conclusion: success
name: sqlite_data
name_is_regexp: true
skip_unpack: false
if_no_artifact_found: ignore
- name: Fetch latest release URL
id: fetch_release_url
run: |
echo "LATEST_RELEASE=$(curl -s "https://api.github.com/repos/stacklok/codegate-ui/releases/latest" | jq -r '.zipball_url')" >> $GITHUB_ENV
- name: Download git lfs dependencies
run: |
git lfs install
git lfs pull
- name: Test build on x86
id: docker_build
uses: docker/build-push-action@67a2d409c0a876cbe6b11854e3e25193efe4e62d # v5
Expand All @@ -34,3 +56,15 @@ jobs:
cache-to: type=gha,mode=max
build-args: |
LATEST_RELEASE=${{ env.LATEST_RELEASE }}
tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
- name: Save Docker image as a tarball
run: |
# List all images
docker images
# Save the image as a tarball
docker save -o image.tar ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
- name: Upload Docker image artifact
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
with:
name: ${{ inputs.artifact-name }}
path: image.tar
141 changes: 141 additions & 0 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
# This workflow will run the integration tests for the project
name: Tests - Integration

on:
workflow_call:
inputs:
artifact-name:
description: 'The name of the artifact to download'
required: true
type: string
secrets:
copilot-key:
description: 'The Copilot key to use for integration tests'
required: true

jobs:
integration-tests:
name: Run
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.12" ]
env:
CONTAINER_NAME: "codegate"
CERT_FILE: "/app/codegate_volume/certs/ca.crt"
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
with:
lfs: true

- name: Ensure file permissions for mounted volume
run: |
mkdir -p ./codegate_volume/certs ./codegate_volume/models ./codegate_volume/db
chmod -R 777 ./codegate_volume
- name: Download Docker image artifact
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
with:
name: ${{ inputs.artifact-name }}

- name: Load Docker image
run: |
docker load -i image.tar
echo "Loaded image:"
docker images
- name: Run container from the loaded image
run: |
# Get the image name
DOCKER_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" | head -n 1)
echo "Running container from image: $DOCKER_IMAGE"
# Run the container
docker run --name $CONTAINER_NAME -d -p 8989:8989 -p 9090:9090 \
-p 8990:8990 \
-v "$(pwd)"/codegate_volume:/app/codegate_volume \
-e CODEGATE_APP_LOG_LEVEL=DEBUG \
--restart unless-stopped $DOCKER_IMAGE
# Confirm the container started
echo "Container started:"
docker ps
# Verify container is running with correct ports
docker ps -f name=$CONTAINER_NAME
# Check mount configuration
docker inspect $CONTAINER_NAME -f '{{ json .Mounts }}' | jq
- name: Ensure certificates are available in the container
timeout-minutes: 4
run: |
# Wait for the cert file to be available in the container
while true; do
echo "Checking for $CERT_FILE in container $CONTAINER_NAME..."
if docker exec "$CONTAINER_NAME" test -f "$CERT_FILE"; then
echo "Cert file found: $CERT_FILE"
break
else
echo "Cert file not found. Retrying in 5 seconds..."
sleep 5
fi
done
# Verify volume contents are accessible
docker exec $CONTAINER_NAME ls -la /app/codegate_volume
# Print the container logs we got so far
docker logs $CONTAINER_NAME
- name: Install the CodeGate certificate
run: |
docker cp codegate:/app/codegate_volume/certs/ca.crt ./codegate.crt
sudo cp ./codegate.crt /usr/local/share/ca-certificates/codegate.crt
sudo update-ca-certificates
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
with:
python-version: ${{ matrix.python-version }}

- name: Install Poetry
uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
with:
version: 2.0.1
virtualenvs-create: true
virtualenvs-in-project: true

- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4
with:
path: .venv
key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}

- name: Install dependencies
run: poetry install --with dev

- name: Run integration tests - Copilot
env:
CODEGATE_PROVIDERS: "copilot"
CA_CERT_FILE: "/home/runner/work/codegate/codegate/codegate_volume/certs/ca.crt"
ENV_COPILOT_KEY: ${{ secrets.copilot-key }}
run: |
poetry run python tests/integration/integration_tests.py
- name: Print the container logs (useful for debugging)
if: always()
run: |
docker logs $CONTAINER_NAME
echo "Models contents:"
ls -la codegate_volume/models
docker exec $CONTAINER_NAME ls -la /app/codegate_volume/models
echo "Certs contents:"
ls -la codegate_volume/certs
docker exec $CONTAINER_NAME ls -la /app/codegate_volume/certs
echo "DB contents:"
ls -la codegate_volume/db
docker exec $CONTAINER_NAME ls -la /app/codegate_volume/db
16 changes: 15 additions & 1 deletion .github/workflows/run-on-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,28 @@
name: Run - pull_request

on:
workflow_dispatch:
pull_request:
branches: [ main ]

jobs:
security:
name: Security Checks
uses: ./.github/workflows/security.yml
ci:
name: Build, Test & Lint
uses: ./.github/workflows/ci.yml
image-build:
name: OCI Image - Build
needs: [ci] # No need to build the image if the CI fails
uses: ./.github/workflows/image-build.yml
with:
artifact-name: "codegate-image"
integration-tests:
name: Integration Tests
needs: [ci, image-build] # We need the image available in order to run the integration tests
uses: ./.github/workflows/integration-tests.yml

with:
artifact-name: "codegate-image"
secrets:
copilot-key: ${{ secrets.INTEGRATION_TESTS_COPILOT_KEY }}
7 changes: 7 additions & 0 deletions .github/workflows/run-on-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,18 @@
name: Run - push_to_main

on:
workflow_dispatch:
push:
branches: [ main ]

jobs:
security:
name: Security Checks
uses: ./.github/workflows/security.yml
ci:
name: Build, Test & Lint
uses: ./.github/workflows/ci.yml
openapi:
name: Generate the OpenAPI docs
needs: [ci]
uses: ./.github/workflows/openapi.yml
3 changes: 2 additions & 1 deletion .github/workflows/run-on-release-published.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ on:
- published

jobs:
openapi:
image-publish:
name: OCI Image - Publish
uses: ./.github/workflows/image-publish.yml
7 changes: 2 additions & 5 deletions .github/workflows/security.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
name: Security

on:
pull_request:
push:
branches:
- main
workflow_call:
schedule:
- cron: '0 0 * * *'

jobs:
dependencies:
runs-on: ubuntu-latest
name: Dependencies & Secrets Scan
name: Dependencies & Secrets Scan via Trivy
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
<img alt="CodeGate logo" src="./static/codegate-logo-dark.svg" width="800px" style="max-width: 100%;">
</picture>

[![CI](https://github.com/stacklok/codegate/actions/workflows/run-on-push.yml/badge.svg)](https://github.com/stacklok/codegate/actions/workflows/run-on-push.yml) | [![](https://dcbadge.vercel.app/api/server/RkzVuTp3WK?logo=discord&label=Discord&color=5865&style=flat)](https://discord.gg/RkzVuTp3WK)
---
[![CI](https://github.com/stacklok/codegate/actions/workflows/run-on-push.yml/badge.svg)](https://github.com/stacklok/codegate/actions/workflows/run-on-push.yml) | [![License: Apache 2.0](https://img.shields.io/badge/License-Apache2.0-brightgreen.svg)](https://opensource.org/licenses/Apache-2.0) | [![](https://dcbadge.vercel.app/api/server/RkzVuTp3WK?logo=discord&label=Discord&color=5865&style=flat)](https://discord.gg/RkzVuTp3WK)

---
## Introduction

<img src="./assets/codegate.gif" style="width: 70%; height: 70%;" />
Expand Down
Binary file modified codegate_volume/models/all-minilm-L6-v2-q5_k_m.gguf
Binary file not shown.
Loading

0 comments on commit 22cc684

Please sign in to comment.