Skip to content

Commit

Permalink
Merge branch 'feature/add-ci-cd' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
arthurcerqueiraindicium authored Apr 15, 2024
2 parents b543825 + a5cc091 commit a8b05b0
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 7 deletions.
66 changes: 66 additions & 0 deletions .github/workflows/integration_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
name: Integration Tests on Databricks Billing

on:
push:
branches:
- feature/add-ci-cd

jobs:
build:
runs-on: ubuntu-latest

env:
SNOWFLAKE_TEST_ACCOUNT: ${{ vars.SNOWFLAKE_TEST_ACCOUNT }}
SNOWFLAKE_TEST_USER: ${{ vars.SNOWFLAKE_TEST_USER }}
SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
SNOWFLAKE_TEST_ROLE: ${{ vars.SNOWFLAKE_TEST_ROLE }}
SNOWFLAKE_TEST_DATABASE: ${{ vars.SNOWFLAKE_TEST_DATABASE }}
SNOWFLAKE_TEST_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_WAREHOUSE }}
SNOWFLAKE_TEST_SCHEMA: ${{ vars.SNOWFLAKE_TEST_SCHEMA }}

DATABRICKS_TEST_CATALOG: ${{ vars.DATABRICKS_TEST_CATALOG }}
DATABRICKS_TEST_SCHEMA: ${{ vars.DATABRICKS_TEST_SCHEMA }}
DATABRICKS_TEST_HOST: ${{ vars.DATABRICKS_TEST_HOST }}
DATABRICKS_TEST_HTTP_PATH: ${{ vars.DATABRICKS_TEST_HTTP_PATH }}
DATABRICKS_TEST_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }}

steps:
- uses: actions/checkout@v2

- name: Set up Python 3.8
uses: actions/setup-python@v2
with:
python-version: 3.8

- name: Install dependencies
run: |
python -m pip install --upgrade pip
cd integration_tests
pip install -r requirements.txt
dbt deps
- name: Integration Test - Snowflake
run: |
cd integration_tests
dbt build --target snowflake --vars "
databricks_billing_database: $SNOWFLAKE_TEST_DATABASE
databricks_billing_schema: $SNOWFLAKE_TEST_SCHEMA
seeds:
databricks_billing:
+database: $SNOWFLAKE_TEST_DATABASE
+schema: $SNOWFLAKE_TEST_SCHEMA
"
continue-on-error: true

- name: Integration Test - Databricks
run: |
cd integration_tests
dbt build --target databricks --vars "
databricks_billing_database: $DATABRICKS_TEST_CATALOG
databricks_billing_schema: $DATABRICKS_TEST_SCHEMA
seeds:
databricks_billing:
+database: $DATABRICKS_TEST_CATALOG
+schema: $DATABRICKS_TEST_SCHEMA
"
continue-on-error: true
4 changes: 2 additions & 2 deletions integration_tests/.gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

venv/
target/
dbt_packages/
logs/
logs/
.env
8 changes: 3 additions & 5 deletions integration_tests/dbt_project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: 'databricks_billing_integration_tests'
version: '1.0.0'
config-version: 2

profile: 'databricks_billing'
profile: 'integration_tests'

model-paths: ["models"]
analysis-paths: ["analyses"]
Expand All @@ -25,10 +25,8 @@ models:

seeds:
databricks_billing:
+database: database_name
+tags: integration_tests
+schema: schema_name

vars:
databricks_billing_database: database_name
databricks_billing_schema: schema_name
databricks_billing_database:
databricks_billing_schema:
22 changes: 22 additions & 0 deletions integration_tests/profiles.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
integration_tests:
target: snowflake
outputs:
snowflake:
type: snowflake
account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}"
user: "{{ env_var('SNOWFLAKE_TEST_USER') }}"
password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}"
role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}"
database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
schema: "{{ env_var('SNOWFLAKE_TEST_SCHEMA') }}"
threads: 5

databricks:
type: databricks
catalog: "{{ env_var('DATABRICKS_TEST_CATALOG') }}"
schema: "{{ env_var('DATABRICKS_TEST_SCHEMA') }}"
host: "{{ env_var('DATABRICKS_TEST_HOST') }}"
http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}"
token: "{{ env_var('DATABRICKS_TEST_TOKEN') }}"
threads: 5

0 comments on commit a8b05b0

Please sign in to comment.