Skip to content

Commit

Permalink
Fail safe catalog errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@buremba
buremba committed Aug 23, 2024
1 parent 7e9860f commit 530ef2e
Show file tree
Hide file tree
Showing 2 changed files with 231 additions and 8 deletions.
227 changes: 222 additions & 5 deletions testing.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 6,
"id": "initial_id",
"metadata": {
"ExecuteTime": {
Expand All @@ -14,13 +14,16 @@
"source": [
"import snowflake.connector\n",
"\n",
"con = snowflake.connector.connect(account='lt51601.europe-west2.gcp', database='MY_CUSTOM_APP', connection_name='default', host='localhostcomputing.com', port='8084', role='ACCOUNTADMIN')\n",
"con = snowflake.connector.connect(account='lt51601.europe-west2.gcp', \n",
" database='my_iceberg_jinjat', connection_name='default',\n",
" host='localhostcomputing.com', \n",
" port='8084', role='ACCOUNTADMIN')\n",
"cur = con.cursor()"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"id": "7165f75bf836dbe3",
"metadata": {
"ExecuteTime": {
Expand All @@ -32,9 +35,223 @@
"outputs_hidden": false
}
},
"outputs": [],
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>created_on</th>\n",
" <th>name</th>\n",
" <th>database_name</th>\n",
" <th>schema_name</th>\n",
" <th>kind</th>\n",
" <th>comment</th>\n",
" <th>cluster_by</th>\n",
" <th>rows</th>\n",
" <th>bytes</th>\n",
" <th>owner</th>\n",
" <th>...</th>\n",
" <th>change_tracking</th>\n",
" <th>is_external</th>\n",
" <th>enable_schema_evolution</th>\n",
" <th>owner_role_type</th>\n",
" <th>is_event</th>\n",
" <th>budget</th>\n",
" <th>is_hybrid</th>\n",
" <th>is_iceberg</th>\n",
" <th>is_dynamic</th>\n",
" <th>is_immutable</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>292278994-08-16 23:12:55.807000-08:00</td>\n",
" <td>CUSTOMER</td>\n",
" <td>MY_ICEBERG_JINJAT</td>\n",
" <td>TPCH_SF1</td>\n",
" <td>TABLE</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>600000</td>\n",
" <td>43210240</td>\n",
" <td>ACCOUNTADMIN</td>\n",
" <td>...</td>\n",
" <td>ON</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" <td>ROLE</td>\n",
" <td>N</td>\n",
" <td>None</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>292278994-08-16 23:12:55.807000-08:00</td>\n",
" <td>DYNAMIC_ICEBERG_TABLE_CUSTOMER</td>\n",
" <td>MY_ICEBERG_JINJAT</td>\n",
" <td>TPCH_SF1</td>\n",
" <td>TABLE</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>600000</td>\n",
" <td>42527232</td>\n",
" <td>ACCOUNTADMIN</td>\n",
" <td>...</td>\n",
" <td>ON</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" <td>ROLE</td>\n",
" <td>N</td>\n",
" <td>None</td>\n",
" <td>N</td>\n",
" <td>Y</td>\n",
" <td>Y</td>\n",
" <td>N</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>292278994-08-16 23:12:55.807000-08:00</td>\n",
" <td>DYNAMIC_TABLE_CUSTOMER</td>\n",
" <td>MY_ICEBERG_JINJAT</td>\n",
" <td>TPCH_SF1</td>\n",
" <td>TABLE</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>600000</td>\n",
" <td>58829312</td>\n",
" <td>ACCOUNTADMIN</td>\n",
" <td>...</td>\n",
" <td>ON</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" <td>ROLE</td>\n",
" <td>N</td>\n",
" <td>None</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" <td>Y</td>\n",
" <td>N</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>292278994-08-16 23:12:55.807000-08:00</td>\n",
" <td>ORDERS_ICEBERG_TPCH_SF1000</td>\n",
" <td>MY_ICEBERG_JINJAT</td>\n",
" <td>TPCH_SF1</td>\n",
" <td>TABLE</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>1500000000</td>\n",
" <td>42525678592</td>\n",
" <td>ACCOUNTADMIN</td>\n",
" <td>...</td>\n",
" <td>OFF</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" <td>ROLE</td>\n",
" <td>N</td>\n",
" <td>None</td>\n",
" <td>N</td>\n",
" <td>Y</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>292278994-08-16 23:12:55.807000-08:00</td>\n",
" <td>TEST_TABLE_MANAGED</td>\n",
" <td>MY_ICEBERG_JINJAT</td>\n",
" <td>TPCH_SF1</td>\n",
" <td>TABLE</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>ACCOUNTADMIN</td>\n",
" <td>...</td>\n",
" <td>OFF</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" <td>ROLE</td>\n",
" <td>N</td>\n",
" <td>None</td>\n",
" <td>N</td>\n",
" <td>Y</td>\n",
" <td>N</td>\n",
" <td>N</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 22 columns</p>\n",
"</div>"
],
"text/plain": [
" created_on name \\\n",
"0 292278994-08-16 23:12:55.807000-08:00 CUSTOMER \n",
"1 292278994-08-16 23:12:55.807000-08:00 DYNAMIC_ICEBERG_TABLE_CUSTOMER \n",
"2 292278994-08-16 23:12:55.807000-08:00 DYNAMIC_TABLE_CUSTOMER \n",
"3 292278994-08-16 23:12:55.807000-08:00 ORDERS_ICEBERG_TPCH_SF1000 \n",
"4 292278994-08-16 23:12:55.807000-08:00 TEST_TABLE_MANAGED \n",
"\n",
" database_name schema_name kind comment cluster_by rows \\\n",
"0 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 600000 \n",
"1 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 600000 \n",
"2 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 600000 \n",
"3 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 1500000000 \n",
"4 MY_ICEBERG_JINJAT TPCH_SF1 TABLE 0 \n",
"\n",
" bytes owner ... change_tracking is_external \\\n",
"0 43210240 ACCOUNTADMIN ... ON N \n",
"1 42527232 ACCOUNTADMIN ... ON N \n",
"2 58829312 ACCOUNTADMIN ... ON N \n",
"3 42525678592 ACCOUNTADMIN ... OFF N \n",
"4 0 ACCOUNTADMIN ... OFF N \n",
"\n",
" enable_schema_evolution owner_role_type is_event budget is_hybrid \\\n",
"0 N ROLE N None N \n",
"1 N ROLE N None N \n",
"2 N ROLE N None N \n",
"3 N ROLE N None N \n",
"4 N ROLE N None N \n",
"\n",
" is_iceberg is_dynamic is_immutable \n",
"0 N N N \n",
"1 Y Y N \n",
"2 N Y N \n",
"3 Y N N \n",
"4 Y N N \n",
"\n",
"[5 rows x 22 columns]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"con.cursor().execute(\"show tables\").fetch_pandas_all()"
"con.cursor().execute(\"show tables in schema tpch_sf1\").fetch_pandas_all()"
]
},
{
Expand Down
12 changes: 9 additions & 3 deletions universql/warehouse/duckdb/duckdb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ def sync_duckdb_catalog(self, locations: typing.Dict[
sqlglot.exp.Table, sqlglot.exp.Expression], ast: sqlglot.exp.Expression) -> Optional[
sqlglot.exp.Expression]:

views = [f"CREATE OR REPLACE VIEW main.\"{table.sql()}\" AS SELECT * FROM {expression.sql()};" for

views = [f"CREATE OR REPLACE VIEW main.{sqlglot.exp.parse_identifier(table.sql())} AS SELECT * FROM {expression.sql()};" for
table, expression in locations.items()]
views_sql = "\n".join(views)
if views:
Expand Down Expand Up @@ -122,8 +123,13 @@ def _do_query(self, raw_query: str) -> (str, List, pyarrow.Table):
local_error_message = (f"Unable to find location of Iceberg tables. "
f"See: https://github.com/buremba/universql#cant-query-native-snowflake-tables. Cause: {e.msg}")

transformed_ast = self.sync_duckdb_catalog(locations,
simplify(ast)) if locations is not None else None
try:
transformed_ast = self.sync_duckdb_catalog(locations,
simplify(ast)) if locations is not None else None
except Exception as e:
transformed_ast = e
local_error_message = f"Unable to sync DuckDB catalog. {str(e)}"

if transformed_ast is None:
last_compute = None
break
Expand Down

0 comments on commit 530ef2e

Please sign in to comment.