Merge branch 'main' into feature/stages_integration

.github/workflows/dwh-benchmark.yml

@@ -0,0 +1,20 @@
+name: deploy
+on: [push]
+jobs:
+  version:
+    name: "Check Snowflake CLI version"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - uses: Snowflake-Labs/[email protected]
+        with:
+          cli-version: "latest"
+      - name: Test project
+        env:
+          SNOWFLAKE_CONNECTIONS_BASE64: ${{ secrets.SNOWFLAKE_CONNECTIONS_BASE64 }}
+        run: |
+          export SNOWFLAKE_CONNECTIONS=$(printf '%s' "$SNOWFLAKE_CONNECTIONS_BASE64" | base64 -d | tr '\r' '\n')
+          snow --version
+          snow connection test
+          snow sql -f /resources/dwh-benchmark/clickbench.sql

DEVELOPMENT.md

@@ -1,3 +1,9 @@
+# Running Github Actions locally
+
+```shell
+act push --container-architecture linux/amd64 --secret-file .env.act --workflows ./.github/workflows/test.yml --insecure-secrets
+```
+
 # [DRAFT] Transformers
 
 If you would like to enable custom logic inside Universql,  to support transition in between different dialects or just 
@@ -74,13 +80,13 @@ For stage integration, something like:
 class SnowflakeStageTransformer(Transformer):
     def __init__(self, source_engine: SnowflakeCatalog, target_engine: DuckDBCatalog):
         super().__init__(source_engine, target_engine)
-        
+
     def transform_sql(self, ast: Expression) -> Expression:
         if isinstance(ast, sqlglot.exp.Var) and ast.name.startswith('@'):
             # transform into full path and create secret on duckdb
             self.target_engine.duckdb.sql("select from stage information_schema.stages where ..")
             if not_exists:
-                self.source_engine.executor().execute_raw("get stage info from fs")
+                self.source_engine.executor().execute_raw(, "get stage info from fs"
                 self.target_engine.duckdb.sql("INSERT INTO information_schema.stages ...")
             return new_ast_with_full_path
         return ast

README.md

@@ -1,4 +1,4 @@
-# Execute Snowflake queries locally on DuckDB
+# 10x more efficient Snowflake compute
 
 UniverSQL is a Snowflake proxy that allows you to run SQL queries **locally** on Snowflake Iceberg tables and Polaris catalog, using DuckDB. 
 You can join Snowflake data with your local datasets, **without any need for a running warehouse**.

examples/sqlglot_visualizer.py → examples/playground/app/sqlglot/page.py

@@ -212,19 +212,19 @@ def ast_to_link_analysis(ast, classes):
 
 default_flattenable_classes = ["Column", "ColumnDef", "Alias"]
 
-flattenable_classes = st.multiselect(
-    "Select what types of expressions you would like to flatten:",
-    sql_expressions.keys(),
-    default=default_flattenable_classes,
-)
-
 if parsed_sql[0]:
+    st.code(parsed_sql[0].__repr__())
+
+    flattenable_classes = st.multiselect(
+        "Select what types of expressions you would like to flatten:",
+        sql_expressions.keys(),
+        default=default_flattenable_classes,
+    )
+
     ast_to_link_analysis(
         parsed_sql[0],
         tuple(
             sql_expressions.get(class_name)
             for class_name in flattenable_classes
         ),
-    )
-
-    st.code(parsed_sql[0].__repr__())
+    )

examples/playground/app/sqlglot/requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+sqlglot
+duckdb
+networkx
+st_link_analysis

examples/playground/universql_project.yml

@@ -0,0 +1,11 @@
+
+name: 'utilities'
+version: '2.0'
+
+profile: 'default'
+config-version: 2
+app-path: 'app'
+target-path: 'target'
+
+vars:
+  example_variable: 'value'