Slugify branch names in dbt CI environment and add macro unit tests

dbt/README.md

@@ -84,3 +84,9 @@ Run tests for only one model:
 ```
 dbt test --select <model_name>
 ```
+
+Run tests for dbt macros:
+
+```
+dbt run-operation test_all
+```

dbt/macros/generate_schema_name.sql

@@ -2,17 +2,33 @@
 -- and replace it with our own namespacing on dev and CI.
 -- See: https://docs.getdbt.com/docs/build/custom-schemas
 {% macro generate_schema_name(custom_schema_name, node) -%}
+    {{ return(_generate_schema_name(
+        custom_schema_name,
+        node,
+        target,
+        env_var,
+        exceptions.raise_compiler_error
+    )) }}
+{%- endmacro %}
 
+-- Helper implementation of generate_schema_name where no arguments
+-- are read from the global variable context. Useful for unit testing.
+{% macro _generate_schema_name(
+    custom_schema_name, node, target, env_var_func, raise_error_func
+) %}
     {#
         According to the dbt docs linked above, this is required to be set by
         the built-in macro that we are overriding, but we don't actually use it
     #}
     {%- set default_schema = target.schema -%}
 
     {%- if target.name == "dev" -%}
-        {%- set schema_prefix = env_var("USER") -%}
+        {%- set schema_prefix = "dev-" ~ env_var_func("USER") ~ "-" -%}
     {%- elif target.name == "ci" -%}
-        {%- set schema_prefix = env_var("GITHUB_HEAD_REF") -%}
+        {%- set github_head_ref = kebab_slugify(
+            env_var_func("GITHUB_HEAD_REF")
+        ) -%}
+        {%- set schema_prefix = "ci-" ~ github_head_ref ~ "-" -%}
     {%- else -%}
         {%- set schema_prefix = "" -%}
     {%- endif -%}
@@ -23,20 +39,19 @@
             The default schema name is not allowed, since we use subdirectory
             organization to map tables/views to their Athena database
         #}
-        {{ exceptions.raise_compiler_error(
+        {{ return(raise_error_func(
             "Missing schema definition for " ~ node.name ~ ". " ~
             "Its containing subdirectory is probably missing a `+schema` " ~
             "attribute under the `models` config in dbt_project.yml."
-        ) }}
+        )) }}
 
     {%- else -%}
 
         {%- set full_schema_name -%}
-            {{ schema_prefix ~ "-" ~ custom_schema_name | trim }}
+            {{ schema_prefix ~ custom_schema_name | trim }}
         {%- endset -%}
 
-        {{ full_schema_name }}
+        {{ return(full_schema_name) }}
 
     {%- endif -%}
-
-{%- endmacro %}
+{% endmacro %}

dbt/macros/slugify.sql

@@ -0,0 +1,16 @@
+-- Variation of dbt_utils.slugify macro using kebab-case instead of snake_case
+{% macro kebab_slugify(string) %}
+    {#- Lower case the string -#}
+    {% set string = string | lower %}
+
+    {#- Replace spaces, slashes, and underscores with hyphens -#}
+    {% set string = modules.re.sub('[ _/]+', '-', string) %}
+
+    {#- Only take letters, numbers, and hyphens -#}
+    {% set string = modules.re.sub('[^a-z0-9-]+', '', string) %}
+
+    {#- Prepends "_" if string begins with a number -#}
+    {% set string = modules.re.sub('^[0-9]', '_' + string[0], string) %}
+
+    {{ return(string) }}
+{% endmacro %}

dbt/macros/tests/test_all.sql

@@ -0,0 +1,6 @@
+-- dbt macro unit testing inspired by this blog post:
+-- https://docs.getdbt.com/blog/unit-testing-dbt-packages
+{% macro test_all() %}
+    {% do test_slugify() %}
+    {% do test_generate_schema_name() %}
+{% endmacro %}

dbt/macros/tests/test_generate_schema_name.sql

@@ -0,0 +1,76 @@
+{% macro test_generate_schema_name() %}
+    {% do test_generate_schema_name_handles_dev_env() %}
+    {% do test_generate_schema_name_handles_ci_env() %}
+    {% do test_generate_schema_name_handles_prod_env() %}
+    {% do test_generate_schema_name_raises_for_default_schema_name() %}
+{% endmacro %}
+
+{% macro mock_env_var(var_name) %}
+    {% if var_name == "USER" %}
+        {{ return("testuser") }}
+    {% elif var_name == "GITHUB_HEAD_REF" %}
+        {{ return("testuser/feature-branch-1") }}
+    {% else %}
+        {{ return("") }}
+    {% endif %}
+{% endmacro %}
+
+{% macro mock_raise_compiler_error(_error) %}
+    {{ return("Compiler error raised") }}
+{% endmacro %}
+
+{% macro test_generate_schema_name_handles_dev_env() %}
+    {% do assert_equals(
+        "test_generate_schema_name_handles_dev_env",
+        _generate_schema_name(
+            "test",
+            {"name": "test"},
+            {"schema": "default", "name": "dev"},
+            mock_env_var,
+            exceptions.raise_compiler_error
+        ),
+        "dev-testuser-test"
+    ) %}
+{% endmacro %}
+
+{% macro test_generate_schema_name_handles_ci_env() %}
+    {% do assert_equals(
+        "test_generate_schema_name_handles_ci_env",
+        _generate_schema_name(
+            "test",
+            {"name": "test"},
+            {"schema": "default", "name": "ci"},
+            mock_env_var,
+            exceptions.raise_compiler_error
+        ),
+        "ci-testuser-feature-branch-1-test"
+    ) %}
+{% endmacro %}
+
+{% macro test_generate_schema_name_handles_prod_env() %}
+    {% do assert_equals(
+        "test_generate_schema_name_handles_prod_env",
+        _generate_schema_name(
+            "test",
+            {"name": "test"},
+            {"schema": "default", "name": "prod"},
+            mock_env_var,
+            exceptions.raise_compiler_error
+        ),
+        "test"
+    ) %}
+{% endmacro %}
+
+{% macro test_generate_schema_name_raises_for_default_schema_name() %}
+    {% do assert_equals(
+        "test_generate_schema_name_raises_for_default_schema_name",
+        _generate_schema_name(
+            None,
+            {"name": "test"},
+            {"schema": "default", "name": "prod"},
+            mock_env_var,
+            mock_raise_compiler_error
+        ),
+        "Compiler error raised"
+    ) %}
+{% endmacro %}

dbt/macros/tests/test_slugify.sql

@@ -0,0 +1,60 @@
+{% macro test_slugify() %}
+    {% do test_kebab_slugify() %}
+{% endmacro %}
+
+{% macro test_kebab_slugify() %}
+    {% do test_kebab_slugify_lowercases_strings() %}
+    {% do test_kebab_slugify_replaces_spaces() %}
+    {% do test_kebab_slugify_replaces_slashes() %}
+    {% do test_kebab_slugify_replaces_underscores() %}
+    {% do test_kebab_slugify_removes_special_characters() %}
+    {% do test_kebab_slugify_handles_leading_numbers() %}
+{% endmacro %}
+
+{% macro test_kebab_slugify_lowercases_strings() %}
+    {{ assert_equals(
+        "test_kebab_slugify_lowercases_strings",
+        kebab_slugify("TEST"),
+        "test"
+    ) }}
+{% endmacro %}
+
+{% macro test_kebab_slugify_replaces_spaces() %}
+    {{ assert_equals(
+        "test_kebab_slugify_replaces_spaces",
+        kebab_slugify("t e s t"),
+        "t-e-s-t"
+    ) }}
+{% endmacro %}
+
+{% macro test_kebab_slugify_replaces_slashes() %}
+    {{ assert_equals(
+        "test_kebab_slugify_replaces_slashes",
+        kebab_slugify("t/e/s/t"),
+        "t-e-s-t"
+    ) }}
+{% endmacro %}
+
+{% macro test_kebab_slugify_replaces_underscores() %}
+    {{ assert_equals(
+        "test_kebab_slugify_replaces_underscores",
+        kebab_slugify("t_e_s_t"),
+        "t-e-s-t"
+    ) }}
+{% endmacro %}
+
+{% macro test_kebab_slugify_removes_special_characters() %}
+    {{ assert_equals(
+        "test_kebab_slugify_removes_special_characters",
+        kebab_slugify("t!@#e$%^s&*()t"),
+        "test"
+    ) }}
+{% endmacro %}
+
+{% macro test_kebab_slugify_handles_leading_numbers() %}
+    {{ assert_equals(
+        "test_kebab_slugify_handles_leading_numbers",
+        kebab_slugify("123test"),
+        "_123test"
+    ) }}
+{% endmacro %}

dbt/macros/tests/utils.sql

@@ -0,0 +1,9 @@
+{% macro assert_equals(test_name, value, expected) %}
+    {% if value == expected %}
+        {% do log(test_name ~ " - PASS", info=True) %}
+    {% else %}
+        {% do exceptions.raise_compiler_error(
+            test_name ~ " - FAIL: " ~ value~ " != " ~ expected
+        ) %}
+    {% endif %}
+{% endmacro %}