Add scalar types to the metadata.

This allows us to make `occurring_scalar_types` private.
hasura · Feb 14, 2024 · 65482a4 · 65482a4
1 parent 6afac55
commit 65482a4
Show file tree

Hide file tree

Showing 16 changed files with 238 additions and 58 deletions.
diff --git a/crates/configuration/src/lib.rs b/crates/configuration/src/lib.rs
@@ -8,4 +8,3 @@ pub use configuration::{
     RawConfiguration, RuntimeConfiguration,
 };
 pub use values::{ConnectionUri, IsolationLevel, PoolSettings, ResolvedSecret};
-pub use version3::occurring_scalar_types;
diff --git a/crates/configuration/src/version3.rs b/crates/configuration/src/version3.rs
@@ -139,6 +139,7 @@ pub async fn configure(
         pool_settings: args.pool_settings,
         isolation_level: args.isolation_level,
         metadata: metadata::Metadata {
+            scalar_types: metadata::ScalarTypes(scalar_types),
             tables,
             native_queries: args.metadata.native_queries,
             aggregate_functions: relevant_aggregate_functions,
@@ -151,7 +152,7 @@ pub async fn configure(
 
 /// Collect all the types that can occur in the metadata. This is a bit circumstantial. A better
 /// approach is likely to record scalar type names directly in the metadata via version2.sql.
-pub fn occurring_scalar_types(
+fn occurring_scalar_types(
     tables: &metadata::TablesInfo,
     native_queries: &metadata::NativeQueries,
 ) -> BTreeSet<metadata::ScalarType> {

diff --git a/crates/connectors/ndc-postgres/src/schema.rs b/crates/connectors/ndc-postgres/src/schema.rs
@@ -21,51 +21,52 @@ pub async fn get_schema(
 ) -> Result<models::SchemaResponse, connector::SchemaError> {
     let configuration::RuntimeConfiguration { metadata, .. } = config;
 
-    let scalar_types: BTreeMap<String, models::ScalarType> =
-        configuration::occurring_scalar_types(&metadata.tables, &metadata.native_queries)
-            .iter()
-            .map(|scalar_type| {
-                (
-                    scalar_type.0.clone(),
-                    models::ScalarType {
-                        aggregate_functions: metadata
-                            .aggregate_functions
-                            .0
-                            .get(scalar_type)
-                            .unwrap_or(&BTreeMap::new())
-                            .iter()
-                            .map(|(function_name, function_definition)| {
-                                (
-                                    function_name.clone(),
-                                    models::AggregateFunctionDefinition {
-                                        result_type: models::Type::Named {
-                                            name: function_definition.return_type.0.clone(),
-                                        },
+    let scalar_types: BTreeMap<String, models::ScalarType> = metadata
+        .scalar_types
+        .0
+        .iter()
+        .map(|scalar_type| {
+            (
+                scalar_type.0.clone(),
+                models::ScalarType {
+                    aggregate_functions: metadata
+                        .aggregate_functions
+                        .0
+                        .get(scalar_type)
+                        .unwrap_or(&BTreeMap::new())
+                        .iter()
+                        .map(|(function_name, function_definition)| {
+                            (
+                                function_name.clone(),
+                                models::AggregateFunctionDefinition {
+                                    result_type: models::Type::Named {
+                                        name: function_definition.return_type.0.clone(),
                                     },
-                                )
-                            })
-                            .collect(),
-                        comparison_operators: metadata
-                            .comparison_operators
-                            .0
-                            .get(scalar_type)
-                            .unwrap_or(&BTreeMap::new())
-                            .iter()
-                            .map(|(op_name, op_def)| {
-                                (
-                                    op_name.clone(),
-                                    models::ComparisonOperatorDefinition::Custom {
-                                        argument_type: models::Type::Named {
-                                            name: op_def.argument_type.0.clone(),
-                                        },
+                                },
+                            )
+                        })
+                        .collect(),
+                    comparison_operators: metadata
+                        .comparison_operators
+                        .0
+                        .get(scalar_type)
+                        .unwrap_or(&BTreeMap::new())
+                        .iter()
+                        .map(|(op_name, op_def)| {
+                            (
+                                op_name.clone(),
+                                models::ComparisonOperatorDefinition::Custom {
+                                    argument_type: models::Type::Named {
+                                        name: op_def.argument_type.0.clone(),
                                     },
-                                )
-                            })
-                            .collect(),
-                    },
-                )
-            })
-            .collect();
+                                },
+                            )
+                        })
+                        .collect(),
+                },
+            )
+        })
+        .collect();
 
     let collections_by_identifier: BTreeMap<(&str, &str), &str> = metadata
         .tables

diff --git a/crates/query-engine/metadata/src/metadata/database.rs b/crates/query-engine/metadata/src/metadata/database.rs
@@ -9,11 +9,16 @@ use std::collections::{BTreeMap, BTreeSet};
 #[serde(rename_all = "camelCase")]
 pub struct CompositeTypes(pub BTreeMap<String, CompositeType>);
 
-/// A Scalar Type.
+/// A scalar type.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct ScalarType(pub String);
 
+/// A set of scalar types.
+#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ScalarTypes(pub BTreeSet<ScalarType>);
+
 /// The type of values that a column, field, or argument may take.
 #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]

diff --git a/crates/query-engine/metadata/src/metadata/mod.rs b/crates/query-engine/metadata/src/metadata/mod.rs
@@ -15,6 +15,8 @@ use serde::{Deserialize, Serialize};
 #[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct Metadata {
+    #[serde(default)]
+    pub scalar_types: ScalarTypes,
     #[serde(default)]
     pub tables: TablesInfo,
     #[serde(default)]

diff --git a/..._tests__postgres__configuration_tests__configuration_tests__get_configuration_schema.snap b/..._tests__postgres__configuration_tests__configuration_tests__get_configuration_schema.snap
@@ -49,6 +49,7 @@ expression: schema
             },
             "metadata": {
               "default": {
+                "scalarTypes": [],
                 "tables": {},
                 "compositeTypes": {},
                 "nativeQueries": {},
@@ -422,6 +423,14 @@ expression: schema
       "description": "Metadata information.",
       "type": "object",
       "properties": {
+        "scalarTypes": {
+          "default": [],
+          "allOf": [
+            {
+              "$ref": "#/definitions/ScalarTypes"
+            }
+          ]
+        },
         "tables": {
           "default": {},
           "allOf": [
@@ -464,6 +473,18 @@ expression: schema
         }
       }
     },
+    "ScalarTypes": {
+      "description": "A set of scalar types.",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/ScalarType"
+      },
+      "uniqueItems": true
+    },
+    "ScalarType": {
+      "description": "A scalar type.",
+      "type": "string"
+    },
     "TablesInfo": {
       "description": "Mapping from a \"table\" name to its information.",
       "type": "object",
@@ -598,10 +619,6 @@ expression: schema
         }
       ]
     },
-    "ScalarType": {
-      "description": "A Scalar Type.",
-      "type": "string"
-    },
     "Nullable": {
       "description": "Can this column contain null values",
       "type": "string",

diff --git a/...__postgres__configuration_tests__configuration_tests__get_rawconfiguration_v3_schema.snap b/...__postgres__configuration_tests__configuration_tests__get_rawconfiguration_v3_schema.snap
@@ -27,6 +27,7 @@ expression: schema
     },
     "metadata": {
       "default": {
+        "scalarTypes": [],
         "tables": {},
         "compositeTypes": {},
         "nativeQueries": {},
@@ -398,6 +399,14 @@ expression: schema
       "description": "Metadata information.",
       "type": "object",
       "properties": {
+        "scalarTypes": {
+          "default": [],
+          "allOf": [
+            {
+              "$ref": "#/definitions/ScalarTypes"
+            }
+          ]
+        },
         "tables": {
           "default": {},
           "allOf": [
@@ -440,6 +449,18 @@ expression: schema
         }
       }
     },
+    "ScalarTypes": {
+      "description": "A set of scalar types.",
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/ScalarType"
+      },
+      "uniqueItems": true
+    },
+    "ScalarType": {
+      "description": "A scalar type.",
+      "type": "string"
+    },
     "TablesInfo": {
       "description": "Mapping from a \"table\" name to its information.",
       "type": "object",
@@ -574,10 +595,6 @@ expression: schema
         }
       ]
     },
-    "ScalarType": {
-      "description": "A Scalar Type.",
-      "type": "string"
-    },
     "Nullable": {
       "description": "Can this column contain null values",
       "type": "string",

diff --git a/...uration_tests__postgres_current_only_configure_v3_initial_configuration_is_unchanged.snap b/...uration_tests__postgres_current_only_configure_v3_initial_configuration_is_unchanged.snap
@@ -10,6 +10,17 @@ expression: default_configuration
   },
   "isolationLevel": "ReadCommitted",
   "metadata": {
+    "scalarTypes": [
+      "bool",
+      "date",
+      "float8",
+      "int4",
+      "int8",
+      "numeric",
+      "text",
+      "timestamp",
+      "varchar"
+    ],
     "tables": {
       "Album": {
         "schemaName": "public",

diff --git a/...shots/databases_tests__postgres__openapi_tests__openapi__up_to_date_generated_schema.snap b/...shots/databases_tests__postgres__openapi_tests__openapi__up_to_date_generated_schema.snap
@@ -37,6 +37,7 @@ expression: generated_schema_json
         },
         "metadata": {
           "default": {
+            "scalarTypes": [],
             "tables": {},
             "compositeTypes": {},
             "nativeQueries": {},
@@ -406,6 +407,14 @@ expression: generated_schema_json
       "description": "Metadata information.",
       "type": "object",
       "properties": {
+        "scalarTypes": {
+          "default": [],
+          "allOf": [
+            {
+              "$ref": "#/components/schemas/ScalarTypes"
+            }
+          ]
+        },
         "tables": {
           "default": {},
           "allOf": [
@@ -448,6 +457,18 @@ expression: generated_schema_json
         }
       }
     },
+    "ScalarTypes": {
+      "description": "A set of scalar types.",
+      "type": "array",
+      "items": {
+        "$ref": "#/components/schemas/ScalarType"
+      },
+      "uniqueItems": true
+    },
+    "ScalarType": {
+      "description": "A scalar type.",
+      "type": "string"
+    },
     "TablesInfo": {
       "description": "Mapping from a \"table\" name to its information.",
       "type": "object",
@@ -578,10 +599,6 @@ expression: generated_schema_json
         }
       ]
     },
-    "ScalarType": {
-      "description": "A Scalar Type.",
-      "type": "string"
-    },
     "Nullable": {
       "description": "Can this column contain null values",
       "type": "string",

diff --git a/justfile b/justfile
@@ -234,6 +234,8 @@ generate-chinook-configuration: build start-dependencies
     echo "$(tput bold)$(tput setaf 3)WARNING:$(tput sgr0) Not updating the Aurora configuration because the connection string is unset."; \
   fi
 
+  ./scripts/archive-old-ndc-metadata.sh '{{POSTGRES_V3_CHINOOK_NDC_METADATA}}'
+
 # start all the databases and Jaeger
 start-dependencies:
   #!/usr/bin/env bash

diff --git a/static/aurora/v3-chinook-ndc-metadata-template.json b/static/aurora/v3-chinook-ndc-metadata-template.json
@@ -13,6 +13,24 @@
   },
   "isolationLevel": "ReadCommitted",
   "metadata": {
+    "scalarTypes": [
+      "bool",
+      "char",
+      "date",
+      "float4",
+      "float8",
+      "int2",
+      "int4",
+      "int8",
+      "numeric",
+      "text",
+      "time",
+      "timestamp",
+      "timestamptz",
+      "timetz",
+      "uuid",
+      "varchar"
+    ],
     "tables": {
       "Album": {
         "schemaName": "public",

diff --git a/static/citus/v3-chinook-ndc-metadata.json b/static/citus/v3-chinook-ndc-metadata.json
@@ -7,6 +7,24 @@
   },
   "isolationLevel": "ReadCommitted",
   "metadata": {
+    "scalarTypes": [
+      "bool",
+      "char",
+      "date",
+      "float4",
+      "float8",
+      "int2",
+      "int4",
+      "int8",
+      "numeric",
+      "text",
+      "time",
+      "timestamp",
+      "timestamptz",
+      "timetz",
+      "uuid",
+      "varchar"
+    ],
     "tables": {
       "Album": {
         "schemaName": "public",