diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml new file mode 100644 index 0000000..2b3ba4a --- /dev/null +++ b/.github/workflows/integration_tests.yml @@ -0,0 +1,66 @@ +name: Integration Tests on Databricks Billing + +on: + push: + branches: + - feature/add-ci-cd + +jobs: + build: + runs-on: ubuntu-latest + + env: + SNOWFLAKE_TEST_ACCOUNT: ${{ vars.SNOWFLAKE_TEST_ACCOUNT }} + SNOWFLAKE_TEST_USER: ${{ vars.SNOWFLAKE_TEST_USER }} + SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} + SNOWFLAKE_TEST_ROLE: ${{ vars.SNOWFLAKE_TEST_ROLE }} + SNOWFLAKE_TEST_DATABASE: ${{ vars.SNOWFLAKE_TEST_DATABASE }} + SNOWFLAKE_TEST_WAREHOUSE: ${{ vars.SNOWFLAKE_TEST_WAREHOUSE }} + SNOWFLAKE_TEST_SCHEMA: ${{ vars.SNOWFLAKE_TEST_SCHEMA }} + + DATABRICKS_TEST_CATALOG: ${{ vars.DATABRICKS_TEST_CATALOG }} + DATABRICKS_TEST_SCHEMA: ${{ vars.DATABRICKS_TEST_SCHEMA }} + DATABRICKS_TEST_HOST: ${{ vars.DATABRICKS_TEST_HOST }} + DATABRICKS_TEST_HTTP_PATH: ${{ vars.DATABRICKS_TEST_HTTP_PATH }} + DATABRICKS_TEST_TOKEN: ${{ secrets.DATABRICKS_TEST_TOKEN }} + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + cd integration_tests + pip install -r requirements.txt + dbt deps + + - name: Integration Test - Snowflake + run: | + cd integration_tests + dbt build --target snowflake --vars " + databricks_billing_database: $SNOWFLAKE_TEST_DATABASE + databricks_billing_schema: $SNOWFLAKE_TEST_SCHEMA + seeds: + databricks_billing: + +database: $SNOWFLAKE_TEST_DATABASE + +schema: $SNOWFLAKE_TEST_SCHEMA + " + continue-on-error: true + + - name: Integration Test - Databricks + run: | + cd integration_tests + dbt build --target databricks --vars " + databricks_billing_database: $DATABRICKS_TEST_CATALOG + databricks_billing_schema: $DATABRICKS_TEST_SCHEMA + seeds: + databricks_billing: + +database: $DATABRICKS_TEST_CATALOG + +schema: $DATABRICKS_TEST_SCHEMA + " + continue-on-error: true \ No newline at end of file diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore index a0c3ed7..ef42b3d 100644 --- a/integration_tests/.gitignore +++ b/integration_tests/.gitignore @@ -1,5 +1,5 @@ - venv/ target/ dbt_packages/ -logs/ \ No newline at end of file +logs/ +.env \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 1a18e24..75ea978 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -2,7 +2,7 @@ name: 'databricks_billing_integration_tests' version: '1.0.0' config-version: 2 -profile: 'databricks_billing' +profile: 'integration_tests' model-paths: ["models"] analysis-paths: ["analyses"] @@ -25,10 +25,8 @@ models: seeds: databricks_billing: - +database: database_name +tags: integration_tests - +schema: schema_name vars: - databricks_billing_database: database_name - databricks_billing_schema: schema_name \ No newline at end of file + databricks_billing_database: + databricks_billing_schema: \ No newline at end of file diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml new file mode 100644 index 0000000..c6da30d --- /dev/null +++ b/integration_tests/profiles.yml @@ -0,0 +1,22 @@ +integration_tests: + target: snowflake + outputs: + snowflake: + type: snowflake + account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" + password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" + role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_TEST_SCHEMA') }}" + threads: 5 + + databricks: + type: databricks + catalog: "{{ env_var('DATABRICKS_TEST_CATALOG') }}" + schema: "{{ env_var('DATABRICKS_TEST_SCHEMA') }}" + host: "{{ env_var('DATABRICKS_TEST_HOST') }}" + http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" + token: "{{ env_var('DATABRICKS_TEST_TOKEN') }}" + threads: 5 \ No newline at end of file